From ca6722ff958ad04e9fb1da474da40ca0d779a693 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 24 Apr 2023 18:41:23 +0200 Subject: [PATCH 001/120] Parse index with single field --- client/index.go | 19 ++ core/parser.go | 8 +- db/schema.go | 2 +- request/graphql/parser.go | 6 +- request/graphql/parser/request.go | 4 +- request/graphql/schema/collection.go | 198 +++++++++++++------- request/graphql/schema/descriptions_test.go | 2 +- request/graphql/schema/index_test.go | 140 ++++++++++++++ 8 files changed, 307 insertions(+), 72 deletions(-) create mode 100644 client/index.go create mode 100644 request/graphql/schema/index_test.go diff --git a/client/index.go b/client/index.go new file mode 100644 index 0000000000..c0df7958ab --- /dev/null +++ b/client/index.go @@ -0,0 +1,19 @@ +package client + +type IndexDirection string + +const ( + Ascending IndexDirection = "ASC" + Descending IndexDirection = "DESC" +) + +type IndexedFieldDescription struct { + Name string + Direction IndexDirection +} + +type IndexDescription struct { + Name string + Fields []IndexedFieldDescription + IsUnique bool +} diff --git a/core/parser.go b/core/parser.go index ee2d2cfbf1..dd238112a0 100644 --- a/core/parser.go +++ b/core/parser.go @@ -50,8 +50,12 @@ type Parser interface { // NewFilterFromString creates a new filter from a string. NewFilterFromString(collectionType string, body string) (immutable.Option[request.Filter], error) - // ParseSDL parses an SDL string into a set of collection descriptions. - ParseSDL(ctx context.Context, schemaString string) ([]client.CollectionDescription, error) + // ParseSDL parses an SDL string into a set of collection descriptions and indexes. + ParseSDL(ctx context.Context, schemaString string) ( + []client.CollectionDescription, + []client.IndexDescription, + error, + ) // Adds the given schema to this parser's model. SetSchema(ctx context.Context, txn datastore.Txn, collections []client.CollectionDescription) error diff --git a/db/schema.go b/db/schema.go index e85b0b6a72..83765994b4 100644 --- a/db/schema.go +++ b/db/schema.go @@ -33,7 +33,7 @@ func (db *db) addSchema( return nil, err } - newDescriptions, err := db.parser.ParseSDL(ctx, schemaString) + newDescriptions, _, err := db.parser.ParseSDL(ctx, schemaString) if err != nil { return nil, err } diff --git a/request/graphql/parser.go b/request/graphql/parser.go index 2f8c910018..75e36f297a 100644 --- a/request/graphql/parser.go +++ b/request/graphql/parser.go @@ -103,7 +103,11 @@ func (p *parser) Parse(ast *ast.Document) (*request.Request, []error) { return query, nil } -func (p *parser) ParseSDL(ctx context.Context, schemaString string) ([]client.CollectionDescription, error) { +func (p *parser) ParseSDL(ctx context.Context, schemaString string) ( + []client.CollectionDescription, + []client.IndexDescription, + error, +) { return schema.FromString(ctx, schemaString) } diff --git a/request/graphql/parser/request.go b/request/graphql/parser/request.go index 31f59b7e2f..c7d7c36140 100644 --- a/request/graphql/parser/request.go +++ b/request/graphql/parser/request.go @@ -109,11 +109,11 @@ func parseDirectives(astDirectives []*ast.Directive) (request.Directives, error) if astDirective.Name.Value == request.ExplainLabel { // Explain directive found, lets parse and validate the directive. - parsedExplainDirctive, err := parseExplainDirective(astDirective) + parsedExplainDirective, err := parseExplainDirective(astDirective) if err != nil { return request.Directives{}, err } - explainDirective = parsedExplainDirctive + explainDirective = parsedExplainDirective } } diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 4058d3edc0..55ffe0775d 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -24,7 +24,11 @@ import ( ) // FromString parses a GQL SDL string into a set of collection descriptions. -func FromString(ctx context.Context, schemaString string) ([]client.CollectionDescription, error) { +func FromString(ctx context.Context, schemaString string) ( + []client.CollectionDescription, + []client.IndexDescription, + error, +) { source := source.NewSource(&source.Source{ Body: []byte(schemaString), }) @@ -35,27 +39,32 @@ func FromString(ctx context.Context, schemaString string) ([]client.CollectionDe }, ) if err != nil { - return nil, err + return nil, nil, err } - desc, err := fromAst(ctx, doc) - return desc, err + return fromAst(ctx, doc) } // fromAst parses a GQL AST into a set of collection descriptions. -func fromAst(ctx context.Context, doc *ast.Document) ([]client.CollectionDescription, error) { +func fromAst(ctx context.Context, doc *ast.Document) ( + []client.CollectionDescription, + []client.IndexDescription, + error, +) { relationManager := NewRelationManager() descriptions := []client.CollectionDescription{} + indexes := []client.IndexDescription{} for _, def := range doc.Definitions { switch defType := def.(type) { case *ast.ObjectDefinition: - description, err := fromAstDefinition(ctx, relationManager, defType) + description, colIndexes, err := fromAstDefinition(ctx, relationManager, defType) if err != nil { - return nil, err + return nil, nil, err } descriptions = append(descriptions, description) + indexes = append(indexes, colIndexes...) default: // Do nothing, ignore it and continue @@ -68,10 +77,10 @@ func fromAst(ctx context.Context, doc *ast.Document) ([]client.CollectionDescrip // after all the collections have been processed. err := finalizeRelations(relationManager, descriptions) if err != nil { - return nil, err + return nil, nil, err } - return descriptions, nil + return descriptions, indexes, nil } // fromAstDefinition parses a AST object definition into a set of collection descriptions. @@ -79,7 +88,7 @@ func fromAstDefinition( ctx context.Context, relationManager *RelationManager, def *ast.ObjectDefinition, -) (client.CollectionDescription, error) { +) (client.CollectionDescription, []client.IndexDescription, error) { fieldDescriptions := []client.FieldDescription{ { Name: request.KeyFieldName, @@ -89,63 +98,12 @@ func fromAstDefinition( } for _, field := range def.Fields { - kind, err := astTypeToKind(field.Type) + tmpFieldsDescriptions, err := fieldsFromAST(field, relationManager, def) if err != nil { - return client.CollectionDescription{}, err + return client.CollectionDescription{}, nil, err } - schema := "" - relationName := "" - relationType := client.RelationType(0) - - if kind == client.FieldKind_FOREIGN_OBJECT || kind == client.FieldKind_FOREIGN_OBJECT_ARRAY { - if kind == client.FieldKind_FOREIGN_OBJECT { - schema = field.Type.(*ast.Named).Name.Value - relationType = client.Relation_Type_ONE - if _, exists := findDirective(field, "primary"); exists { - relationType |= client.Relation_Type_Primary - } - - // An _id field is added for every 1-N relationship from this object. - fieldDescriptions = append(fieldDescriptions, client.FieldDescription{ - Name: fmt.Sprintf("%s_id", field.Name.Value), - Kind: client.FieldKind_DocKey, - Typ: defaultCRDTForFieldKind[client.FieldKind_DocKey], - RelationType: client.Relation_Type_INTERNAL_ID, - }) - } else if kind == client.FieldKind_FOREIGN_OBJECT_ARRAY { - schema = field.Type.(*ast.List).Type.(*ast.Named).Name.Value - relationType = client.Relation_Type_MANY - } - - relationName, err = getRelationshipName(field, def.Name.Value, schema) - if err != nil { - return client.CollectionDescription{}, err - } - - // Register the relationship so that the relationship manager can evaluate - // relationsip properties dependent on both collections in the relationship. - _, err := relationManager.RegisterSingle( - relationName, - schema, - field.Name.Value, - relationType, - ) - if err != nil { - return client.CollectionDescription{}, err - } - } - - fieldDescription := client.FieldDescription{ - Name: field.Name.Value, - Kind: kind, - Typ: defaultCRDTForFieldKind[kind], - Schema: schema, - RelationName: relationName, - RelationType: relationType, - } - - fieldDescriptions = append(fieldDescriptions, fieldDescription) + fieldDescriptions = append(fieldDescriptions, tmpFieldsDescriptions...) } // sort the fields lexicographically @@ -159,13 +117,123 @@ func fromAstDefinition( return fieldDescriptions[i].Name < fieldDescriptions[j].Name }) + indexDescriptions := []client.IndexDescription{} + for _, directive := range def.Directives { + if directive.Name.Value == "index" { + index, err := indexFromAST(directive) + if err != nil { + return client.CollectionDescription{}, nil, err + } + indexDescriptions = append(indexDescriptions, index) + } + } + return client.CollectionDescription{ Name: def.Name.Value, Schema: client.SchemaDescription{ Name: def.Name.Value, Fields: fieldDescriptions, }, - }, nil + }, indexDescriptions, nil +} + +func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { + desc := client.IndexDescription{} + var directions *ast.ListValue + for _, arg := range directive.Arguments { + switch arg.Name.Value { + case "name": + desc.Name = arg.Value.(*ast.StringValue).Value + case "fields": + for _, field := range arg.Value.(*ast.ListValue).Values { + desc.Fields = append(desc.Fields, client.IndexedFieldDescription{ + Name: field.(*ast.StringValue).Value, + }) + break + } + case "directions": + directions = arg.Value.(*ast.ListValue) + case "unique": + desc.IsUnique = arg.Value.(*ast.BooleanValue).Value + } + } + if directions != nil { + dirVal := directions.Values[0].(*ast.EnumValue).Value + if dirVal == "ASC" { + desc.Fields[0].Direction = client.Ascending + } else if dirVal == "DESC" { + desc.Fields[0].Direction = client.Descending + } + } else { + desc.Fields[0].Direction = client.Ascending + } + return desc, nil +} + +func fieldsFromAST(field *ast.FieldDefinition, + relationManager *RelationManager, + def *ast.ObjectDefinition, +) ([]client.FieldDescription, error) { + kind, err := astTypeToKind(field.Type) + if err != nil { + return nil, err + } + + schema := "" + relationName := "" + relationType := client.RelationType(0) + + fieldDescriptions := []client.FieldDescription{} + + if kind == client.FieldKind_FOREIGN_OBJECT || kind == client.FieldKind_FOREIGN_OBJECT_ARRAY { + if kind == client.FieldKind_FOREIGN_OBJECT { + schema = field.Type.(*ast.Named).Name.Value + relationType = client.Relation_Type_ONE + if _, exists := findDirective(field, "primary"); exists { + relationType |= client.Relation_Type_Primary + } + + // An _id field is added for every 1-N relationship from this object. + fieldDescriptions = append(fieldDescriptions, client.FieldDescription{ + Name: fmt.Sprintf("%s_id", field.Name.Value), + Kind: client.FieldKind_DocKey, + Typ: defaultCRDTForFieldKind[client.FieldKind_DocKey], + RelationType: client.Relation_Type_INTERNAL_ID, + }) + } else if kind == client.FieldKind_FOREIGN_OBJECT_ARRAY { + schema = field.Type.(*ast.List).Type.(*ast.Named).Name.Value + relationType = client.Relation_Type_MANY + } + + relationName, err = getRelationshipName(field, def.Name.Value, schema) + if err != nil { + return nil, err + } + + // Register the relationship so that the relationship manager can evaluate + // relationsip properties dependent on both collections in the relationship. + _, err := relationManager.RegisterSingle( + relationName, + schema, + field.Name.Value, + relationType, + ) + if err != nil { + return nil, err + } + } + + fieldDescription := client.FieldDescription{ + Name: field.Name.Value, + Kind: kind, + Typ: defaultCRDTForFieldKind[kind], + Schema: schema, + RelationName: relationName, + RelationType: relationType, + } + + fieldDescriptions = append(fieldDescriptions, fieldDescription) + return fieldDescriptions, nil } func astTypeToKind(t ast.Type) (client.FieldKind, error) { diff --git a/request/graphql/schema/descriptions_test.go b/request/graphql/schema/descriptions_test.go index 21fecd2009..0132d24291 100644 --- a/request/graphql/schema/descriptions_test.go +++ b/request/graphql/schema/descriptions_test.go @@ -581,7 +581,7 @@ func TestSingleSimpleType(t *testing.T) { func runCreateDescriptionTest(t *testing.T, testcase descriptionTestCase) { ctx := context.Background() - descs, err := FromString(ctx, testcase.sdl) + descs, _, err := FromString(ctx, testcase.sdl) assert.NoError(t, err, testcase.description) assert.Equal(t, len(descs), len(testcase.targetDescs), testcase.description) diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go new file mode 100644 index 0000000000..816debe931 --- /dev/null +++ b/request/graphql/schema/index_test.go @@ -0,0 +1,140 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package schema + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/sourcenetwork/defradb/client" +) + +func TestSingleIndex(t *testing.T) { + cases := []indexTestCase{ + { + description: "Index with a single field", + sdl: ` + type user @index(fields: ["name"]) { + name: String + } + `, + targetDescriptions: []client.IndexDescription{ + { + Name: "", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + IsUnique: false, + }, + }, + }, + { + description: "Index with a name", + sdl: ` + type user @index(name: "userIndex", fields: ["name"]) { + name: String + } + `, + targetDescriptions: []client.IndexDescription{ + { + Name: "userIndex", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + }, + }, + }, + { + description: "Unique index", + sdl: ` + type user @index(fields: ["name"], unique: true) { + name: String + } + `, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + IsUnique: true, + }, + }, + }, + { + description: "Index explicitly not unique", + sdl: ` + type user @index(fields: ["name"], unique: false) { + name: String + } + `, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + IsUnique: false, + }, + }, + }, + { + description: "Index with explicit ascending field", + sdl: ` + type user @index(fields: ["name"], directions: [ASC]) { + name: String + } + `, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}}, + }, + }, + }, + { + description: "Index with descending field", + sdl: ` + type user @index(fields: ["name"], directions: [DESC]) { + name: String + } + `, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Descending}}, + }, + }, + }, + } + + for _, test := range cases { + parseIndexAndTest(t, test) + } +} + +func parseIndexAndTest(t *testing.T, testCase indexTestCase) { + ctx := context.Background() + + _, indexes, err := FromString(ctx, testCase.sdl) + assert.NoError(t, err, testCase.description) + assert.Equal(t, len(indexes), len(testCase.targetDescriptions), testCase.description) + + for i, d := range indexes { + assert.Equal(t, testCase.targetDescriptions[i], d, testCase.description) + } +} + +type indexTestCase struct { + description string + sdl string + targetDescriptions []client.IndexDescription +} From 989fca8845ca4146b581e0860e0980a764d94dd0 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 25 Apr 2023 15:47:55 +0200 Subject: [PATCH 002/120] Add some error checking while parsing --- request/graphql/schema/collection.go | 57 ++++++++++-- request/graphql/schema/errors.go | 6 ++ request/graphql/schema/index_test.go | 130 +++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 7 deletions(-) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 55ffe0775d..0d052e027b 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -137,6 +137,22 @@ func fromAstDefinition( }, indexDescriptions, nil } +func isValidIndexName(name string) bool { + if len(name) == 0 { + return false + } + if name[0] != '_' && (name[0] < 'a' || name[0] > 'z') && (name[0] < 'A' || name[0] > 'Z') { + return false + } + for i := 1; i < len(name); i++ { + c := name[i] + if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && c != '_' { + return false + } + } + return true +} + func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { desc := client.IndexDescription{} var directions *ast.ListValue @@ -144,24 +160,51 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { switch arg.Name.Value { case "name": desc.Name = arg.Value.(*ast.StringValue).Value + if !isValidIndexName(desc.Name) { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } case "fields": - for _, field := range arg.Value.(*ast.ListValue).Values { + fieldsVal, ok := arg.Value.(*ast.ListValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } + for _, field := range fieldsVal.Values { + fieldVal, ok := field.(*ast.StringValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } desc.Fields = append(desc.Fields, client.IndexedFieldDescription{ - Name: field.(*ast.StringValue).Value, + Name: fieldVal.Value, }) break } case "directions": - directions = arg.Value.(*ast.ListValue) + var ok bool + directions, ok = arg.Value.(*ast.ListValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } case "unique": - desc.IsUnique = arg.Value.(*ast.BooleanValue).Value + if boolVal, ok := arg.Value.(*ast.BooleanValue); !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } else { + desc.IsUnique = boolVal.Value + } + default: + return client.IndexDescription{}, ErrIndexWithUnknownArg } } + if len(desc.Fields) == 0 { + return client.IndexDescription{}, ErrIndexMissingFields + } if directions != nil { - dirVal := directions.Values[0].(*ast.EnumValue).Value - if dirVal == "ASC" { + dirVal, ok := directions.Values[0].(*ast.EnumValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } + if dirVal.Value == "ASC" { desc.Fields[0].Direction = client.Ascending - } else if dirVal == "DESC" { + } else if dirVal.Value == "DESC" { desc.Fields[0].Direction = client.Descending } } else { diff --git a/request/graphql/schema/errors.go b/request/graphql/schema/errors.go index 8720a04cd7..9290436ee5 100644 --- a/request/graphql/schema/errors.go +++ b/request/graphql/schema/errors.go @@ -22,6 +22,9 @@ const ( errTypeNotFound string = "no type found for given name" errRelationNotFound string = "no relation found" errNonNullForTypeNotSupported string = "NonNull variants for type are not supported" + errIndexMissingFields string = "index missing fields" + errIndexUnknownArgument string = "index with unknown argument" + errIndexInvalidArgument string = "index with invalid argument" ) var ( @@ -41,6 +44,9 @@ var ( // NonNull is the literal name of the GQL type, so we have to disable the linter //nolint:revive ErrNonNullNotSupported = errors.New("NonNull fields are not currently supported") + ErrIndexMissingFields = errors.New(errIndexMissingFields) + ErrIndexWithUnknownArg = errors.New(errIndexUnknownArgument) + ErrIndexWithInvalidArg = errors.New(errIndexInvalidArgument) ) func NewErrDuplicateField(objectName, fieldName string) error { diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index 816debe931..f3190496bd 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -121,6 +121,123 @@ func TestSingleIndex(t *testing.T) { } } +func TestInvalidIndexSyntax(t *testing.T) { + cases := []invalidIndexTestCase{ + { + description: "missing 'fields' argument", + sdl: ` + type user @index(name: "userIndex", unique: true) { + name: String + } + `, + expectedErr: errIndexMissingFields, + }, + { + description: "unknown argument", + sdl: ` + type user @index(unknown: "something", fields: ["name"]) { + name: String + } + `, + expectedErr: errIndexUnknownArgument, + }, + { + description: "index name starts with a number", + sdl: ` + type user @index(name: "1_user_name", fields: ["name"]) { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "index with empty name", + sdl: ` + type user @index(name: "", fields: ["name"]) { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "index name with spaces", + sdl: ` + type user @index(name: "user name", fields: ["name"]) { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "index name with special symbols", + sdl: ` + type user @index(name: "user!name", fields: ["name"]) { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "invalid 'unique' value type", + sdl: ` + type user @index(fields: ["name"], unique: "true") { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "invalid 'fields' value type (not a list)", + sdl: ` + type user @index(fields: "name") { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "invalid 'fields' value type (not a string list)", + sdl: ` + type user @index(fields: [1]) { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "invalid 'directions' value type (not a list)", + sdl: ` + type user @index(fields: ["name"], directions: "ASC") { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "invalid 'directions' value type (not a string list)", + sdl: ` + type user @index(fields: ["name"], directions: [1]) { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + { + description: "invalid 'directions' value type (invalid element value)", + sdl: ` + type user @index(fields: ["name"], directions: ["direction"]) { + name: String + } + `, + expectedErr: errIndexInvalidArgument, + }, + } + + for _, test := range cases { + parseInvalidIndexAndTest(t, test) + } +} + func parseIndexAndTest(t *testing.T, testCase indexTestCase) { ctx := context.Background() @@ -133,8 +250,21 @@ func parseIndexAndTest(t *testing.T, testCase indexTestCase) { } } +func parseInvalidIndexAndTest(t *testing.T, testCase invalidIndexTestCase) { + ctx := context.Background() + + _, _, err := FromString(ctx, testCase.sdl) + assert.EqualError(t, err, testCase.expectedErr, testCase.description) +} + type indexTestCase struct { description string sdl string targetDescriptions []client.IndexDescription } + +type invalidIndexTestCase struct { + description string + sdl string + expectedErr string +} From e2e52502c44bc3e4e2e31c34b1f45c5c7f85a037 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 25 Apr 2023 16:00:29 +0200 Subject: [PATCH 003/120] Parse multiple fields for composite index --- request/graphql/schema/collection.go | 23 +++-- request/graphql/schema/index_test.go | 132 +++++++++------------------ 2 files changed, 55 insertions(+), 100 deletions(-) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 0d052e027b..434f2aeac3 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -176,7 +176,6 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { desc.Fields = append(desc.Fields, client.IndexedFieldDescription{ Name: fieldVal.Value, }) - break } case "directions": var ok bool @@ -198,17 +197,21 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { return client.IndexDescription{}, ErrIndexMissingFields } if directions != nil { - dirVal, ok := directions.Values[0].(*ast.EnumValue) - if !ok { - return client.IndexDescription{}, ErrIndexWithInvalidArg - } - if dirVal.Value == "ASC" { - desc.Fields[0].Direction = client.Ascending - } else if dirVal.Value == "DESC" { - desc.Fields[0].Direction = client.Descending + for i := range desc.Fields { + dirVal, ok := directions.Values[i].(*ast.EnumValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } + if dirVal.Value == "ASC" { + desc.Fields[i].Direction = client.Ascending + } else if dirVal.Value == "DESC" { + desc.Fields[i].Direction = client.Descending + } } } else { - desc.Fields[0].Direction = client.Ascending + for i := range desc.Fields { + desc.Fields[i].Direction = client.Ascending + } } return desc, nil } diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index f3190496bd..cad01d3322 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -23,11 +23,7 @@ func TestSingleIndex(t *testing.T) { cases := []indexTestCase{ { description: "Index with a single field", - sdl: ` - type user @index(fields: ["name"]) { - name: String - } - `, + sdl: `type user @index(fields: ["name"]) {}`, targetDescriptions: []client.IndexDescription{ { Name: "", @@ -40,11 +36,7 @@ func TestSingleIndex(t *testing.T) { }, { description: "Index with a name", - sdl: ` - type user @index(name: "userIndex", fields: ["name"]) { - name: String - } - `, + sdl: `type user @index(name: "userIndex", fields: ["name"]) {}`, targetDescriptions: []client.IndexDescription{ { Name: "userIndex", @@ -56,11 +48,7 @@ func TestSingleIndex(t *testing.T) { }, { description: "Unique index", - sdl: ` - type user @index(fields: ["name"], unique: true) { - name: String - } - `, + sdl: `type user @index(fields: ["name"], unique: true) {}`, targetDescriptions: []client.IndexDescription{ { Fields: []client.IndexedFieldDescription{ @@ -72,11 +60,7 @@ func TestSingleIndex(t *testing.T) { }, { description: "Index explicitly not unique", - sdl: ` - type user @index(fields: ["name"], unique: false) { - name: String - } - `, + sdl: `type user @index(fields: ["name"], unique: false) {}`, targetDescriptions: []client.IndexDescription{ { Fields: []client.IndexedFieldDescription{ @@ -88,11 +72,7 @@ func TestSingleIndex(t *testing.T) { }, { description: "Index with explicit ascending field", - sdl: ` - type user @index(fields: ["name"], directions: [ASC]) { - name: String - } - `, + sdl: `type user @index(fields: ["name"], directions: [ASC]) {}`, targetDescriptions: []client.IndexDescription{ { Fields: []client.IndexedFieldDescription{ @@ -102,11 +82,7 @@ func TestSingleIndex(t *testing.T) { }, { description: "Index with descending field", - sdl: ` - type user @index(fields: ["name"], directions: [DESC]) { - name: String - } - `, + sdl: `type user @index(fields: ["name"], directions: [DESC]) {}`, targetDescriptions: []client.IndexDescription{ { Fields: []client.IndexedFieldDescription{ @@ -114,6 +90,30 @@ func TestSingleIndex(t *testing.T) { }, }, }, + { + description: "Index with 2 fields", + sdl: `type user @index(fields: ["name", "age"]) {}`, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + {Name: "age", Direction: client.Ascending}, + }, + }, + }, + }, + { + description: "Index with 2 fields and 2 directions", + sdl: `type user @index(fields: ["name", "age"], directions: [ASC, DESC]) {}`, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + {Name: "age", Direction: client.Descending}, + }, + }, + }, + }, } for _, test := range cases { @@ -125,110 +125,62 @@ func TestInvalidIndexSyntax(t *testing.T) { cases := []invalidIndexTestCase{ { description: "missing 'fields' argument", - sdl: ` - type user @index(name: "userIndex", unique: true) { - name: String - } - `, + sdl: `type user @index(name: "userIndex", unique: true) {}`, expectedErr: errIndexMissingFields, }, { description: "unknown argument", - sdl: ` - type user @index(unknown: "something", fields: ["name"]) { - name: String - } - `, + sdl: `type user @index(unknown: "something", fields: ["name"]) {}`, expectedErr: errIndexUnknownArgument, }, { description: "index name starts with a number", - sdl: ` - type user @index(name: "1_user_name", fields: ["name"]) { - name: String - } - `, + sdl: `type user @index(name: "1_user_name", fields: ["name"]) {}`, expectedErr: errIndexInvalidArgument, }, { description: "index with empty name", - sdl: ` - type user @index(name: "", fields: ["name"]) { - name: String - } - `, + sdl: `type user @index(name: "", fields: ["name"]) {}`, expectedErr: errIndexInvalidArgument, }, { description: "index name with spaces", - sdl: ` - type user @index(name: "user name", fields: ["name"]) { - name: String - } - `, + sdl: `type user @index(name: "user name", fields: ["name"]) {}`, expectedErr: errIndexInvalidArgument, }, { description: "index name with special symbols", - sdl: ` - type user @index(name: "user!name", fields: ["name"]) { - name: String - } - `, + sdl: `type user @index(name: "user!name", fields: ["name"]) {}`, expectedErr: errIndexInvalidArgument, }, { description: "invalid 'unique' value type", - sdl: ` - type user @index(fields: ["name"], unique: "true") { - name: String - } - `, + sdl: `type user @index(fields: ["name"], unique: "true") {}`, expectedErr: errIndexInvalidArgument, }, { description: "invalid 'fields' value type (not a list)", - sdl: ` - type user @index(fields: "name") { - name: String - } - `, + sdl: `type user @index(fields: "name") {}`, expectedErr: errIndexInvalidArgument, }, { description: "invalid 'fields' value type (not a string list)", - sdl: ` - type user @index(fields: [1]) { - name: String - } - `, + sdl: `type user @index(fields: [1]) {}`, expectedErr: errIndexInvalidArgument, }, { description: "invalid 'directions' value type (not a list)", - sdl: ` - type user @index(fields: ["name"], directions: "ASC") { - name: String - } - `, + sdl: `type user @index(fields: ["name"], directions: "ASC") {}`, expectedErr: errIndexInvalidArgument, }, { description: "invalid 'directions' value type (not a string list)", - sdl: ` - type user @index(fields: ["name"], directions: [1]) { - name: String - } - `, + sdl: `type user @index(fields: ["name"], directions: [1]) {}`, expectedErr: errIndexInvalidArgument, }, { description: "invalid 'directions' value type (invalid element value)", - sdl: ` - type user @index(fields: ["name"], directions: ["direction"]) { - name: String - } - `, + sdl: `type user @index(fields: ["name"], directions: ["direction"]) {}`, expectedErr: errIndexInvalidArgument, }, } From 71f72cf838798cb3c0e36231f603b9fe61777ba2 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 25 Apr 2023 16:31:11 +0200 Subject: [PATCH 004/120] Add some more parsing checks --- request/graphql/schema/collection.go | 15 +++++++++++---- request/graphql/schema/index_test.go | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 434f2aeac3..2e9c71c878 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -159,7 +159,11 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { for _, arg := range directive.Arguments { switch arg.Name.Value { case "name": - desc.Name = arg.Value.(*ast.StringValue).Value + nameVal, ok := arg.Value.(*ast.StringValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } + desc.Name = nameVal.Value if !isValidIndexName(desc.Name) { return client.IndexDescription{}, ErrIndexWithInvalidArg } @@ -184,11 +188,11 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { return client.IndexDescription{}, ErrIndexWithInvalidArg } case "unique": - if boolVal, ok := arg.Value.(*ast.BooleanValue); !ok { + boolVal, ok := arg.Value.(*ast.BooleanValue) + if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg - } else { - desc.IsUnique = boolVal.Value } + desc.IsUnique = boolVal.Value default: return client.IndexDescription{}, ErrIndexWithUnknownArg } @@ -197,6 +201,9 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { return client.IndexDescription{}, ErrIndexMissingFields } if directions != nil { + if len(directions.Values) != len(desc.Fields) { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } for i := range desc.Fields { dirVal, ok := directions.Values[i].(*ast.EnumValue) if !ok { diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index cad01d3322..24d60a8187 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -133,6 +133,11 @@ func TestInvalidIndexSyntax(t *testing.T) { sdl: `type user @index(unknown: "something", fields: ["name"]) {}`, expectedErr: errIndexUnknownArgument, }, + { + description: "invalid index name type", + sdl: `type user @index(name: 1, fields: ["name"]) {}`, + expectedErr: errIndexInvalidArgument, + }, { description: "index name starts with a number", sdl: `type user @index(name: "1_user_name", fields: ["name"]) {}`, @@ -183,6 +188,16 @@ func TestInvalidIndexSyntax(t *testing.T) { sdl: `type user @index(fields: ["name"], directions: ["direction"]) {}`, expectedErr: errIndexInvalidArgument, }, + { + description: "fewer directions than fields", + sdl: `type user @index(fields: ["name", "age"], directions: [ASC]) {}`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "more directions than fields", + sdl: `type user @index(fields: ["name"], directions: [ASC, DESC]) {}`, + expectedErr: errIndexInvalidArgument, + }, } for _, test := range cases { From f1c96afaff08dfc64a2c669cc765f12942d504b9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 26 Apr 2023 09:46:59 +0200 Subject: [PATCH 005/120] Return slice of slices for index descriptions --- core/parser.go | 2 +- request/graphql/parser.go | 2 +- request/graphql/schema/collection.go | 8 ++++---- tests/bench/query/planner/utils.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/parser.go b/core/parser.go index dd238112a0..b7c567cf26 100644 --- a/core/parser.go +++ b/core/parser.go @@ -53,7 +53,7 @@ type Parser interface { // ParseSDL parses an SDL string into a set of collection descriptions and indexes. ParseSDL(ctx context.Context, schemaString string) ( []client.CollectionDescription, - []client.IndexDescription, + [][]client.IndexDescription, error, ) diff --git a/request/graphql/parser.go b/request/graphql/parser.go index 75e36f297a..f6a9d19425 100644 --- a/request/graphql/parser.go +++ b/request/graphql/parser.go @@ -105,7 +105,7 @@ func (p *parser) Parse(ast *ast.Document) (*request.Request, []error) { func (p *parser) ParseSDL(ctx context.Context, schemaString string) ( []client.CollectionDescription, - []client.IndexDescription, + [][]client.IndexDescription, error, ) { return schema.FromString(ctx, schemaString) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 2e9c71c878..04b9dd2586 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -26,7 +26,7 @@ import ( // FromString parses a GQL SDL string into a set of collection descriptions. func FromString(ctx context.Context, schemaString string) ( []client.CollectionDescription, - []client.IndexDescription, + [][]client.IndexDescription, error, ) { source := source.NewSource(&source.Source{ @@ -48,12 +48,12 @@ func FromString(ctx context.Context, schemaString string) ( // fromAst parses a GQL AST into a set of collection descriptions. func fromAst(ctx context.Context, doc *ast.Document) ( []client.CollectionDescription, - []client.IndexDescription, + [][]client.IndexDescription, error, ) { relationManager := NewRelationManager() descriptions := []client.CollectionDescription{} - indexes := []client.IndexDescription{} + indexes := [][]client.IndexDescription{} for _, def := range doc.Definitions { switch defType := def.(type) { @@ -64,7 +64,7 @@ func fromAst(ctx context.Context, doc *ast.Document) ( } descriptions = append(descriptions, description) - indexes = append(indexes, colIndexes...) + indexes = append(indexes, colIndexes) default: // Do nothing, ignore it and continue diff --git a/tests/bench/query/planner/utils.go b/tests/bench/query/planner/utils.go index 148347aa2f..5841cb460a 100644 --- a/tests/bench/query/planner/utils.go +++ b/tests/bench/query/planner/utils.go @@ -107,7 +107,7 @@ func buildParser( return nil, err } - collectionDescriptions, err := gqlSchema.FromString(ctx, schema) + collectionDescriptions, _, err := gqlSchema.FromString(ctx, schema) if err != nil { return nil, err } From 44c8e05396aa71253f4c774e02138a29c0ff5da6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 26 Apr 2023 12:28:05 +0200 Subject: [PATCH 006/120] parse field index --- request/graphql/schema/collection.go | 45 ++++++++- request/graphql/schema/index_test.go | 143 ++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 6 deletions(-) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 04b9dd2586..d90d044847 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -97,6 +97,7 @@ func fromAstDefinition( }, } + indexDescriptions := []client.IndexDescription{} for _, field := range def.Fields { tmpFieldsDescriptions, err := fieldsFromAST(field, relationManager, def) if err != nil { @@ -104,6 +105,16 @@ func fromAstDefinition( } fieldDescriptions = append(fieldDescriptions, tmpFieldsDescriptions...) + + for _, directive := range field.Directives { + if directive.Name.Value == "index" { + index, err := fieldIndexFromAST(field, directive) + if err != nil { + return client.CollectionDescription{}, nil, err + } + indexDescriptions = append(indexDescriptions, index) + } + } } // sort the fields lexicographically @@ -117,7 +128,6 @@ func fromAstDefinition( return fieldDescriptions[i].Name < fieldDescriptions[j].Name }) - indexDescriptions := []client.IndexDescription{} for _, directive := range def.Directives { if directive.Name.Value == "index" { index, err := indexFromAST(directive) @@ -153,6 +163,39 @@ func isValidIndexName(name string) bool { return true } +func fieldIndexFromAST(field *ast.FieldDefinition, directive *ast.Directive) (client.IndexDescription, error) { + desc := client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + { + Name: field.Name.Value, + Direction: client.Ascending, + }, + }, + } + for _, arg := range directive.Arguments { + switch arg.Name.Value { + case "name": + nameVal, ok := arg.Value.(*ast.StringValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } + desc.Name = nameVal.Value + if !isValidIndexName(desc.Name) { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } + case "unique": + boolVal, ok := arg.Value.(*ast.BooleanValue) + if !ok { + return client.IndexDescription{}, ErrIndexWithInvalidArg + } + desc.IsUnique = boolVal.Value + default: + return client.IndexDescription{}, ErrIndexWithUnknownArg + } + } + return desc, nil +} + func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { desc := client.IndexDescription{} var directions *ast.ListValue diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index 24d60a8187..54a89a06d6 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -19,7 +19,7 @@ import ( "github.com/sourcenetwork/defradb/client" ) -func TestSingleIndex(t *testing.T) { +func TestStructIndex(t *testing.T) { cases := []indexTestCase{ { description: "Index with a single field", @@ -121,7 +121,7 @@ func TestSingleIndex(t *testing.T) { } } -func TestInvalidIndexSyntax(t *testing.T) { +func TestInvalidStructIndex(t *testing.T) { cases := []invalidIndexTestCase{ { description: "missing 'fields' argument", @@ -205,14 +205,147 @@ func TestInvalidIndexSyntax(t *testing.T) { } } +func TestFieldIndex(t *testing.T) { + cases := []indexTestCase{ + { + description: "field index", + sdl: `type user { + name: String @index + }`, + targetDescriptions: []client.IndexDescription{ + { + Name: "", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + IsUnique: false, + }, + }, + }, + { + description: "field index with name", + sdl: `type user { + name: String @index(name: "nameIndex") + }`, + targetDescriptions: []client.IndexDescription{ + { + Name: "nameIndex", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + IsUnique: false, + }, + }, + }, + { + description: "unique field index", + sdl: `type user { + name: String @index(unique: true) + }`, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + IsUnique: true, + }, + }, + }, + { + description: "field index explicitly not unique", + sdl: `type user { + name: String @index(unique: false) + }`, + targetDescriptions: []client.IndexDescription{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + IsUnique: false, + }, + }, + }, + } + + for _, test := range cases { + parseIndexAndTest(t, test) + } +} + +func TestInvalidFieldIndex(t *testing.T) { + cases := []invalidIndexTestCase{ + { + description: "forbidden 'field' argument", + sdl: `type user { + name: String @index(field: "name") + }`, + expectedErr: errIndexUnknownArgument, + }, + { + description: "forbidden 'direction' argument", + sdl: `type user { + name: String @index(direction: ASC) + }`, + expectedErr: errIndexUnknownArgument, + }, + { + description: "invalid field index name type", + sdl: `type user { + name: String @index(name: 1) + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "field index name starts with a number", + sdl: `type user { + name: String @index(name: "1_user_name") + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "field index with empty name", + sdl: `type user { + name: String @index(name: "") + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "field index name with spaces", + sdl: `type user { + name: String @index(name: "user name") + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "field index name with special symbols", + sdl: `type user { + name: String @index(name: "user!name") + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "invalid 'unique' value type", + sdl: `type user { + name: String @index(unique: "true") + }`, + expectedErr: errIndexInvalidArgument, + }, + } + + for _, test := range cases { + parseInvalidIndexAndTest(t, test) + } +} + func parseIndexAndTest(t *testing.T, testCase indexTestCase) { ctx := context.Background() - _, indexes, err := FromString(ctx, testCase.sdl) + _, colIndexes, err := FromString(ctx, testCase.sdl) assert.NoError(t, err, testCase.description) - assert.Equal(t, len(indexes), len(testCase.targetDescriptions), testCase.description) + assert.Equal(t, len(colIndexes), 1, testCase.description) + assert.Equal(t, len(colIndexes[0]), len(testCase.targetDescriptions), testCase.description) - for i, d := range indexes { + for i, d := range colIndexes[0] { assert.Equal(t, testCase.targetDescriptions[i], d, testCase.description) } } From cdca6ad25948a8e09ba56c2ce2f549bec9a69826 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 1 May 2023 09:55:12 +0200 Subject: [PATCH 007/120] Add validation of some index properties --- client/collection.go | 9 +++ db/collection.go | 11 +++ db/errors.go | 108 ++++++++++++++------------ db/index.go | 105 ++++++++++++++++++++++++++ db/index_test.go | 176 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 360 insertions(+), 49 deletions(-) create mode 100644 db/index.go create mode 100644 db/index_test.go diff --git a/client/collection.go b/client/collection.go index f59bf43d6b..c1f5e821cf 100644 --- a/client/collection.go +++ b/client/collection.go @@ -136,6 +136,15 @@ type Collection interface { // GetAllDocKeys returns all the document keys that exist in the collection. GetAllDocKeys(ctx context.Context) (<-chan DocKeysResult, error) + + // CreateIndex creates a new index on the collection. + CreateIndex(ctx context.Context, desc IndexDescription) (IndexDescription, error) + + // DropIndex drops an index from the collection. + DropIndex(ctx context.Context, indexName string) error + + // GetIndexes returns all the indexes that exist on the collection. + GetIndexes(ctx context.Context) ([]IndexDescription, error) } // DocKeysResult wraps the result of an attempt at a DocKey retrieval operation. diff --git a/db/collection.go b/db/collection.go index 5810a4745b..778435410e 100644 --- a/db/collection.go +++ b/db/collection.go @@ -189,6 +189,17 @@ func (db *db) createCollection( return col, nil } +// createCollectionIndex creates a new collection index and saves it to the database in its system store. +func (db *db) createCollectionIndex( + ctx context.Context, + txn datastore.Txn, + collectionName string, + desc client.IndexDescription, +) (client.IndexDescription, error) { + col, _ := db.getCollectionByName(ctx, txn, collectionName) + return col.CreateIndex(ctx, desc) +} + // updateCollection updates the persisted collection description matching the name of the given // description, to the values in the given description. // diff --git a/db/errors.go b/db/errors.go index 7642ca7253..34b8ed335d 100644 --- a/db/errors.go +++ b/db/errors.go @@ -16,27 +16,32 @@ import ( ) const ( - errFailedToGetHeads string = "failed to get document heads" - errFailedToCreateCollectionQuery string = "failed to create collection prefix query" - errFailedToGetCollection string = "failed to get collection" - errDocVerification string = "the document verification failed" - errAddingP2PCollection string = "cannot add collection ID" - errRemovingP2PCollection string = "cannot remove collection ID" - errAddCollectionWithPatch string = "unknown collection, adding collections via patch is not supported" - errCollectionIDDoesntMatch string = "CollectionID does not match existing" - errSchemaIDDoesntMatch string = "SchemaID does not match existing" - errCannotModifySchemaName string = "modifying the schema name is not supported" - errCannotSetVersionID string = "setting the VersionID is not supported. It is updated automatically" - errCannotSetFieldID string = "explicitly setting a field ID value is not supported" - errCannotAddRelationalField string = "the adding of new relation fields is not yet supported" - errDuplicateField string = "duplicate field" - errCannotMutateField string = "mutating an existing field is not supported" - errCannotMoveField string = "moving fields is not currently supported" - errInvalidCRDTType string = "only default or LWW (last writer wins) CRDT types are supported" - errCannotDeleteField string = "deleting an existing field is not supported" - errFieldKindNotFound string = "no type found for given name" + errFailedToGetHeads string = "failed to get document heads" + errFailedToCreateCollectionQuery string = "failed to create collection prefix query" + errFailedToGetCollection string = "failed to get collection" + errDocVerification string = "the document verification failed" + errAddingP2PCollection string = "cannot add collection ID" + errRemovingP2PCollection string = "cannot remove collection ID" + errAddCollectionWithPatch string = "unknown collection, adding collections via patch is not supported" + errCollectionIDDoesntMatch string = "CollectionID does not match existing" + errSchemaIDDoesntMatch string = "SchemaID does not match existing" + errCannotModifySchemaName string = "modifying the schema name is not supported" + errCannotSetVersionID string = "setting the VersionID is not supported. It is updated automatically" + errCannotSetFieldID string = "explicitly setting a field ID value is not supported" + errCannotAddRelationalField string = "the adding of new relation fields is not yet supported" + errDuplicateField string = "duplicate field" + errCannotMutateField string = "mutating an existing field is not supported" + errCannotMoveField string = "moving fields is not currently supported" + errInvalidCRDTType string = "only default or LWW (last writer wins) CRDT types are supported" + errCannotDeleteField string = "deleting an existing field is not supported" + errFieldKindNotFound string = "no type found for given name" errDocumentAlreadyExists string = "a document with the given dockey already exists" errDocumentDeleted string = "a document with the given dockey has been deleted" + errIndexMissingFields string = "index missing fields" + errIndexFieldMissingName string = "index field missing name" + errIndexFieldMissingDirection string = "index field missing direction" + errIndexSingleFieldWrongDirection string = "wrong direction for index with a single field" + errIndexWithNameAlreadyExists string = "index with name already exists" ) var ( @@ -54,36 +59,41 @@ var ( ErrInvalidMergeValueType = errors.New( "the type of value in the merge patch doesn't match the schema", ) - ErrMissingDocFieldToUpdate = errors.New("missing document field to update") - ErrDocMissingKey = errors.New("document is missing key") - ErrMergeSubTypeNotSupported = errors.New("merge doesn't support sub types yet") - ErrInvalidFilter = errors.New("invalid filter") - ErrInvalidOpPath = errors.New("invalid patch op path") - ErrDocumentAlreadyExists = errors.New(errDocumentAlreadyExists) - ErrDocumentDeleted = errors.New(errDocumentDeleted) - ErrUnknownCRDTArgument = errors.New("invalid CRDT arguments") - ErrUnknownCRDT = errors.New("unknown crdt") - ErrSchemaFirstFieldDocKey = errors.New("collection schema first field must be a DocKey") - ErrCollectionAlreadyExists = errors.New("collection already exists") - ErrCollectionNameEmpty = errors.New("collection name can't be empty") - ErrSchemaIdEmpty = errors.New("schema ID can't be empty") - ErrSchemaVersionIdEmpty = errors.New("schema version ID can't be empty") - ErrKeyEmpty = errors.New("key cannot be empty") - ErrAddingP2PCollection = errors.New(errAddingP2PCollection) - ErrRemovingP2PCollection = errors.New(errRemovingP2PCollection) - ErrAddCollectionWithPatch = errors.New(errAddCollectionWithPatch) - ErrCollectionIDDoesntMatch = errors.New(errCollectionIDDoesntMatch) - ErrSchemaIDDoesntMatch = errors.New(errSchemaIDDoesntMatch) - ErrCannotModifySchemaName = errors.New(errCannotModifySchemaName) - ErrCannotSetVersionID = errors.New(errCannotSetVersionID) - ErrCannotSetFieldID = errors.New(errCannotSetFieldID) - ErrCannotAddRelationalField = errors.New(errCannotAddRelationalField) - ErrDuplicateField = errors.New(errDuplicateField) - ErrCannotMutateField = errors.New(errCannotMutateField) - ErrCannotMoveField = errors.New(errCannotMoveField) - ErrInvalidCRDTType = errors.New(errInvalidCRDTType) - ErrCannotDeleteField = errors.New(errCannotDeleteField) - ErrFieldKindNotFound = errors.New(errFieldKindNotFound) + ErrMissingDocFieldToUpdate = errors.New("missing document field to update") + ErrDocMissingKey = errors.New("document is missing key") + ErrMergeSubTypeNotSupported = errors.New("merge doesn't support sub types yet") + ErrInvalidFilter = errors.New("invalid filter") + ErrInvalidOpPath = errors.New("invalid patch op path") + ErrDocumentAlreadyExists = errors.New(errDocumentAlreadyExists) + ErrDocumentDeleted = errors.New(errDocumentDeleted) + ErrUnknownCRDTArgument = errors.New("invalid CRDT arguments") + ErrUnknownCRDT = errors.New("unknown crdt") + ErrSchemaFirstFieldDocKey = errors.New("collection schema first field must be a DocKey") + ErrCollectionAlreadyExists = errors.New("collection already exists") + ErrCollectionNameEmpty = errors.New("collection name can't be empty") + ErrSchemaIdEmpty = errors.New("schema ID can't be empty") + ErrSchemaVersionIdEmpty = errors.New("schema version ID can't be empty") + ErrKeyEmpty = errors.New("key cannot be empty") + ErrAddingP2PCollection = errors.New(errAddingP2PCollection) + ErrRemovingP2PCollection = errors.New(errRemovingP2PCollection) + ErrAddCollectionWithPatch = errors.New(errAddCollectionWithPatch) + ErrCollectionIDDoesntMatch = errors.New(errCollectionIDDoesntMatch) + ErrSchemaIDDoesntMatch = errors.New(errSchemaIDDoesntMatch) + ErrCannotModifySchemaName = errors.New(errCannotModifySchemaName) + ErrCannotSetVersionID = errors.New(errCannotSetVersionID) + ErrCannotSetFieldID = errors.New(errCannotSetFieldID) + ErrCannotAddRelationalField = errors.New(errCannotAddRelationalField) + ErrDuplicateField = errors.New(errDuplicateField) + ErrCannotMutateField = errors.New(errCannotMutateField) + ErrCannotMoveField = errors.New(errCannotMoveField) + ErrInvalidCRDTType = errors.New(errInvalidCRDTType) + ErrCannotDeleteField = errors.New(errCannotDeleteField) + ErrFieldKindNotFound = errors.New(errFieldKindNotFound) + ErrIndexMissingFields = errors.New(errIndexMissingFields) + ErrIndexFieldMissingName = errors.New(errIndexFieldMissingName) + ErrIndexFieldMissingDirection = errors.New(errIndexFieldMissingDirection) + ErrIndexSingleFieldWrongDirection = errors.New(errIndexSingleFieldWrongDirection) + ErrIndexWithNameAlreadyExists = errors.New(errIndexWithNameAlreadyExists) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document diff --git a/db/index.go b/db/index.go new file mode 100644 index 0000000000..9793c7b54d --- /dev/null +++ b/db/index.go @@ -0,0 +1,105 @@ +package db + +import ( + "context" + "strings" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/core" +) + +type CollectionIndex interface { + Save(core.DataStoreKey, client.Value) error + Name() string + Description() client.IndexDescription +} + +func NewCollectionIndex( + collection client.Collection, + desc client.IndexDescription, +) CollectionIndex { + return &collectionSimpleIndex{collection: collection, desc: desc} +} + +type collectionSimpleIndex struct { + collection client.Collection + desc client.IndexDescription +} + +func (c *collectionSimpleIndex) Save(core.DataStoreKey, client.Value) error { + return nil +} + +func (c *collectionSimpleIndex) Name() string { + return c.desc.Name +} + +func (c *collectionSimpleIndex) Description() client.IndexDescription { + return c.desc +} + +func validateIndexDescriptionFields(fields []client.IndexedFieldDescription) error { + if len(fields) == 0 { + return ErrIndexMissingFields + } + if len(fields) == 1 && fields[0].Direction == client.Descending { + return ErrIndexSingleFieldWrongDirection + } + for i := range fields { + if fields[i].Name == "" { + return ErrIndexFieldMissingName + } + if fields[i].Direction == "" { + fields[i].Direction = client.Ascending + } + } + return nil +} + +func generateIndexName(col client.Collection, fields []client.IndexedFieldDescription) string { + sb := strings.Builder{} + direction := "ASC" + //if fields[0].Direction == client.Descending { + //direction = "DESC" + //} + sb.WriteString(strings.ToLower(col.Name())) + sb.WriteByte('_') + sb.WriteString(strings.ToLower(fields[0].Name)) + sb.WriteByte('_') + sb.WriteString(direction) + return sb.String() +} + +func (c *collection) CreateIndex( + ctx context.Context, + desc client.IndexDescription, +) (client.IndexDescription, error) { + index, err := c.createIndex(ctx, desc) + if err != nil { + return client.IndexDescription{}, err + } + return index.Description(), nil +} + +func (c *collection) DropIndex(ctx context.Context, indexName string) error { + return nil +} + +func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { + return nil, nil +} + +func (c *collection) createIndex( + ctx context.Context, + desc client.IndexDescription, +) (CollectionIndex, error) { + err := validateIndexDescriptionFields(desc.Fields) + if err != nil { + return nil, err + } + if desc.Name == "" { + desc.Name = generateIndexName(c, desc.Fields) + } + colIndex := NewCollectionIndex(c, desc) + return colIndex, nil +} diff --git a/db/index_test.go b/db/index_test.go new file mode 100644 index 0000000000..2fdde4a460 --- /dev/null +++ b/db/index_test.go @@ -0,0 +1,176 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package db + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/datastore" +) + +type indexTestFixture struct { + ctx context.Context + db *implicitTxnDB + txn datastore.Txn + collection client.Collection +} + +func (f *indexTestFixture) createCollectionIndex( + desc client.IndexDescription, +) (client.IndexDescription, error) { + return f.db.createCollectionIndex(f.ctx, f.txn, f.collection.Name(), desc) +} + +func newIndexTestFixture(t *testing.T) *indexTestFixture { + ctx := context.Background() + db, err := newMemoryDB(ctx) + assert.NoError(t, err) + txn, err := db.NewTxn(ctx, false) + assert.NoError(t, err) + + desc := client.CollectionDescription{ + Name: "Users", + Schema: client.SchemaDescription{ + Fields: []client.FieldDescription{ + { + Name: "_key", + Kind: client.FieldKind_DocKey, + }, + { + Name: "name", + Kind: client.FieldKind_STRING, + Typ: client.LWW_REGISTER, + }, + { + Name: "age", + Kind: client.FieldKind_INT, + Typ: client.LWW_REGISTER, + }, + { + Name: "weight", + Kind: client.FieldKind_FLOAT, + Typ: client.LWW_REGISTER, + }, + }, + }, + } + + col, err := db.createCollection(ctx, txn, desc) + assert.NoError(t, err) + + //err = txn.Commit(ctx) + //assert.NoError(t, err) + + return &indexTestFixture{ + ctx: ctx, + db: db, + txn: txn, + collection: col, + } + +} + +func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + _, err := f.createCollectionIndex(client.IndexDescription{ + Name: "some_index_name", + }) + assert.EqualError(t, err, errIndexMissingFields) +} + +func TestCreateIndex_IfValidInput_CreateIndex(t *testing.T) { + f := newIndexTestFixture(t) + + desc := client.IndexDescription{ + Name: "some_index_name", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + } + resultDesc, err := f.createCollectionIndex(desc) + assert.NoError(t, err) + assert.Equal(t, resultDesc.Name, desc.Name) + assert.Equal(t, resultDesc, desc) +} + +func TestCreateIndex_IfFieldNameIsEmpty_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + desc := client.IndexDescription{ + Name: "some_index_name", + Fields: []client.IndexedFieldDescription{ + {Name: "", Direction: client.Ascending}, + }, + } + _, err := f.createCollectionIndex(desc) + assert.EqualError(t, err, errIndexFieldMissingName) +} + +func TestCreateIndex_IfFieldHasNoDirection_DefaultToAsc(t *testing.T) { + f := newIndexTestFixture(t) + + desc := client.IndexDescription{ + Name: "some_index_name", + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + newDesc, err := f.createCollectionIndex(desc) + assert.NoError(t, err) + assert.Equal(t, newDesc.Fields[0].Direction, client.Ascending) +} + +func TestCreateIndex_IfNameIsNotSpecified_GenerateWithLowerCase(t *testing.T) { + f := newIndexTestFixture(t) + + desc := client.IndexDescription{ + Name: "", + Fields: []client.IndexedFieldDescription{ + {Name: "Name", Direction: client.Ascending}, + }, + } + f.collection.Description().Schema.Fields[1].Name = "Name" + newDesc, _ := f.createCollectionIndex(desc) + assert.Equal(t, newDesc.Name, "users_name_ASC") +} + +func TestCreateIndex_IfSingleFieldInDescOrder_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + desc := client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Descending}, + }, + } + _, err := f.createCollectionIndex(desc) + assert.EqualError(t, err, errIndexSingleFieldWrongDirection) +} + +func TestCreateIndex_IndexWithNameAlreadyExists_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + name := "some_index_name" + desc1 := client.IndexDescription{ + Name: name, + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + desc2 := client.IndexDescription{ + Name: name, + Fields: []client.IndexedFieldDescription{{Name: "age"}}, + } + _, err := f.createCollectionIndex(desc1) + assert.NoError(t, err) + _, err = f.createCollectionIndex(desc2) + assert.EqualError(t, err, errIndexWithNameAlreadyExists) +} From b68e41f6b216bf66f629882fddbfcb94a2233824 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 1 May 2023 17:17:16 +0200 Subject: [PATCH 008/120] Store collection index description --- client/index.go | 6 ++--- core/key.go | 33 ++++++++++++++++++++++++++++ db/collection.go | 3 ++- db/index.go | 25 +++++++++++++++++++++ request/graphql/schema/collection.go | 4 ++-- request/graphql/schema/index_test.go | 14 ++++++------ 6 files changed, 72 insertions(+), 13 deletions(-) diff --git a/client/index.go b/client/index.go index c0df7958ab..453033ed9d 100644 --- a/client/index.go +++ b/client/index.go @@ -13,7 +13,7 @@ type IndexedFieldDescription struct { } type IndexDescription struct { - Name string - Fields []IndexedFieldDescription - IsUnique bool + Name string + Fields []IndexedFieldDescription + Unique bool } diff --git a/core/key.go b/core/key.go index 756290a607..f21281b4fc 100644 --- a/core/key.go +++ b/core/key.go @@ -44,6 +44,7 @@ const ( COLLECTION = "/collection/names" COLLECTION_SCHEMA = "/collection/schema" COLLECTION_SCHEMA_VERSION = "/collection/version" + COLLECTION_INDEX = "/collection/index" SEQ = "/seq" PRIMARY_KEY = "/pk" REPLICATOR = "/replicator/id" @@ -106,6 +107,13 @@ type CollectionSchemaVersionKey struct { var _ Key = (*CollectionSchemaVersionKey)(nil) +type CollectionIndexKey struct { + CollectionID string + IndexName string +} + +var _ Key = (*CollectionIndexKey)(nil) + type P2PCollectionKey struct { CollectionID string } @@ -210,6 +218,10 @@ func NewCollectionSchemaVersionKey(schemaVersionId string) CollectionSchemaVersi return CollectionSchemaVersionKey{SchemaVersionId: schemaVersionId} } +func NewCollectionIndexKey(colID, name string) CollectionIndexKey { + return CollectionIndexKey{CollectionID: colID} +} + func NewSequenceKey(name string) SequenceKey { return SequenceKey{SequenceName: name} } @@ -401,6 +413,27 @@ func (k CollectionSchemaVersionKey) ToDS() ds.Key { return ds.NewKey(k.ToString()) } +func (k CollectionIndexKey) ToString() string { + result := COLLECTION_INDEX + + if k.CollectionID != "" { + result = result + "/" + k.CollectionID + } + if k.IndexName != "" { + result = result + "/" + k.IndexName + } + + return result +} + +func (k CollectionIndexKey) Bytes() []byte { + return []byte(k.ToString()) +} + +func (k CollectionIndexKey) ToDS() ds.Key { + return ds.NewKey(k.ToString()) +} + func (k SequenceKey) ToString() string { result := SEQ diff --git a/db/collection.go b/db/collection.go index 778435410e..e4ac4163eb 100644 --- a/db/collection.go +++ b/db/collection.go @@ -196,7 +196,8 @@ func (db *db) createCollectionIndex( collectionName string, desc client.IndexDescription, ) (client.IndexDescription, error) { - col, _ := db.getCollectionByName(ctx, txn, collectionName) + col, _ := db.getCollectionByName(ctx, txn, collectionName) // TODO: test error + col = col.WithTxn(txn) return col.CreateIndex(ctx, desc) } diff --git a/db/index.go b/db/index.go index 9793c7b54d..828a4c98a4 100644 --- a/db/index.go +++ b/db/index.go @@ -2,6 +2,7 @@ package db import ( "context" + "encoding/json" "strings" "github.com/sourcenetwork/defradb/client" @@ -100,6 +101,30 @@ func (c *collection) createIndex( if desc.Name == "" { desc.Name = generateIndexName(c, desc.Fields) } + + txn, err := c.getTxn(ctx, false) + if err != nil { + return nil, err + } + + indexKey := core.NewCollectionIndexKey(c.Name(), desc.Name) + exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) + if err != nil { + return nil, err + } + if exists { + return nil, ErrIndexWithNameAlreadyExists + } + + buf, err := json.Marshal(desc) + if err != nil { + return nil, err + } + + err = txn.Systemstore().Put(ctx, indexKey.ToDS(), buf) + if err != nil { + return nil, err + } colIndex := NewCollectionIndex(c, desc) return colIndex, nil } diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index d90d044847..ef55177156 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -188,7 +188,7 @@ func fieldIndexFromAST(field *ast.FieldDefinition, directive *ast.Directive) (cl if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg } - desc.IsUnique = boolVal.Value + desc.Unique = boolVal.Value default: return client.IndexDescription{}, ErrIndexWithUnknownArg } @@ -235,7 +235,7 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg } - desc.IsUnique = boolVal.Value + desc.Unique = boolVal.Value default: return client.IndexDescription{}, ErrIndexWithUnknownArg } diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index 54a89a06d6..3e53975da4 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -30,7 +30,7 @@ func TestStructIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - IsUnique: false, + Unique: false, }, }, }, @@ -54,7 +54,7 @@ func TestStructIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - IsUnique: true, + Unique: true, }, }, }, @@ -66,7 +66,7 @@ func TestStructIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - IsUnique: false, + Unique: false, }, }, }, @@ -218,7 +218,7 @@ func TestFieldIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - IsUnique: false, + Unique: false, }, }, }, @@ -233,7 +233,7 @@ func TestFieldIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - IsUnique: false, + Unique: false, }, }, }, @@ -247,7 +247,7 @@ func TestFieldIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - IsUnique: true, + Unique: true, }, }, }, @@ -261,7 +261,7 @@ func TestFieldIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - IsUnique: false, + Unique: false, }, }, }, From 02036a1dd8a0c8813c5f0495bc440cfd7461ebba Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 2 May 2023 09:15:55 +0200 Subject: [PATCH 009/120] Generate unique index name --- core/key.go | 2 +- db/index.go | 57 ++++++++++++++++++++++++++++++++++++++---------- db/index_test.go | 27 ++++++++++++++++++++++- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/core/key.go b/core/key.go index f21281b4fc..57baf856ee 100644 --- a/core/key.go +++ b/core/key.go @@ -219,7 +219,7 @@ func NewCollectionSchemaVersionKey(schemaVersionId string) CollectionSchemaVersi } func NewCollectionIndexKey(colID, name string) CollectionIndexKey { - return CollectionIndexKey{CollectionID: colID} + return CollectionIndexKey{CollectionID: colID, IndexName: name} } func NewSequenceKey(name string) SequenceKey { diff --git a/db/index.go b/db/index.go index 828a4c98a4..84f829bba5 100644 --- a/db/index.go +++ b/db/index.go @@ -3,6 +3,7 @@ package db import ( "context" "encoding/json" + "strconv" "strings" "github.com/sourcenetwork/defradb/client" @@ -57,7 +58,7 @@ func validateIndexDescriptionFields(fields []client.IndexedFieldDescription) err return nil } -func generateIndexName(col client.Collection, fields []client.IndexedFieldDescription) string { +func generateIndexName(col client.Collection, fields []client.IndexedFieldDescription, inc int) string { sb := strings.Builder{} direction := "ASC" //if fields[0].Direction == client.Descending { @@ -68,6 +69,10 @@ func generateIndexName(col client.Collection, fields []client.IndexedFieldDescri sb.WriteString(strings.ToLower(fields[0].Name)) sb.WriteByte('_') sb.WriteString(direction) + if inc > 1 { + sb.WriteByte('_') + sb.WriteString(strconv.Itoa(inc)) + } return sb.String() } @@ -98,25 +103,18 @@ func (c *collection) createIndex( if err != nil { return nil, err } - if desc.Name == "" { - desc.Name = generateIndexName(c, desc.Fields) - } - txn, err := c.getTxn(ctx, false) + indexKey, err := c.processIndexName(ctx, &desc) if err != nil { return nil, err } - indexKey := core.NewCollectionIndexKey(c.Name(), desc.Name) - exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) + buf, err := json.Marshal(desc) if err != nil { return nil, err } - if exists { - return nil, ErrIndexWithNameAlreadyExists - } - buf, err := json.Marshal(desc) + txn, err := c.getTxn(ctx, false) if err != nil { return nil, err } @@ -128,3 +126,40 @@ func (c *collection) createIndex( colIndex := NewCollectionIndex(c, desc) return colIndex, nil } + +func (c *collection) processIndexName( + ctx context.Context, + desc *client.IndexDescription, +) (core.CollectionIndexKey, error) { + txn, err := c.getTxn(ctx, true) + if err != nil { + return core.CollectionIndexKey{}, err + } + + var indexKey core.CollectionIndexKey + if desc.Name == "" { + nameIncrement := 1 + for { + desc.Name = generateIndexName(c, desc.Fields, nameIncrement) + indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) + exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) + if err != nil { + return core.CollectionIndexKey{}, err + } + if !exists { + break + } + nameIncrement++ + } + } else { + indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) + exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) + if err != nil { + return core.CollectionIndexKey{}, err + } + if exists { + return core.CollectionIndexKey{}, ErrIndexWithNameAlreadyExists + } + } + return indexKey, nil +} diff --git a/db/index_test.go b/db/index_test.go index 2fdde4a460..54dbcd259f 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -157,7 +157,7 @@ func TestCreateIndex_IfSingleFieldInDescOrder_ReturnError(t *testing.T) { assert.EqualError(t, err, errIndexSingleFieldWrongDirection) } -func TestCreateIndex_IndexWithNameAlreadyExists_ReturnError(t *testing.T) { +func TestCreateIndex_IfIndexWithNameAlreadyExists_ReturnError(t *testing.T) { f := newIndexTestFixture(t) name := "some_index_name" @@ -174,3 +174,28 @@ func TestCreateIndex_IndexWithNameAlreadyExists_ReturnError(t *testing.T) { _, err = f.createCollectionIndex(desc2) assert.EqualError(t, err, errIndexWithNameAlreadyExists) } + +func TestCreateIndex_IfGeneratedNameMatchesExisting_AddIncrement(t *testing.T) { + f := newIndexTestFixture(t) + + name := "users_age_ASC" + desc1 := client.IndexDescription{ + Name: name, + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + desc2 := client.IndexDescription{ + Name: name + "_2", + Fields: []client.IndexedFieldDescription{{Name: "weight"}}, + } + desc3 := client.IndexDescription{ + Name: "", + Fields: []client.IndexedFieldDescription{{Name: "age"}}, + } + _, err := f.createCollectionIndex(desc1) + assert.NoError(t, err) + _, err = f.createCollectionIndex(desc2) + assert.NoError(t, err) + newDesc3, err := f.createCollectionIndex(desc3) + assert.NoError(t, err) + assert.Equal(t, newDesc3.Name, name+"_3") +} From 9374bbbaa396ee51b1dc4265772945a0feef8f6c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 2 May 2023 10:32:17 +0200 Subject: [PATCH 010/120] Test storing to system storage --- db/index_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/db/index_test.go b/db/index_test.go index 54dbcd259f..92d5c51cbc 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -12,11 +12,13 @@ package db import ( "context" + "encoding/json" "testing" "github.com/stretchr/testify/assert" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" ) @@ -199,3 +201,23 @@ func TestCreateIndex_IfGeneratedNameMatchesExisting_AddIncrement(t *testing.T) { assert.NoError(t, err) assert.Equal(t, newDesc3.Name, name+"_3") } + +func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { + f := newIndexTestFixture(t) + + name := "users_age_ASC" + desc := client.IndexDescription{ + Name: name, + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + _, err := f.createCollectionIndex(desc) + assert.NoError(t, err) + + key := core.NewCollectionIndexKey(f.collection.Name(), name) + data, err := f.txn.Systemstore().Get(f.ctx, key.ToDS()) + assert.NoError(t, err) + var deserialized client.IndexDescription + err = json.Unmarshal(data, &deserialized) + assert.NoError(t, err) + assert.Equal(t, deserialized, desc) +} From 36932ea00a9ad1b37e6900d6dab158d535e03f76 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 2 May 2023 16:22:32 +0200 Subject: [PATCH 011/120] Add tests for index key --- core/key.go | 6 +++--- core/key_test.go | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/core/key.go b/core/key.go index 57baf856ee..e800347cf3 100644 --- a/core/key.go +++ b/core/key.go @@ -418,9 +418,9 @@ func (k CollectionIndexKey) ToString() string { if k.CollectionID != "" { result = result + "/" + k.CollectionID - } - if k.IndexName != "" { - result = result + "/" + k.IndexName + if k.IndexName != "" { + result = result + "/" + k.IndexName + } } return result diff --git a/core/key_test.go b/core/key_test.go index 865ece9c26..2e1ad63cad 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -107,3 +107,23 @@ func TestNewDataStoreKey_GivenAStringWithExtraSuffix(t *testing.T) { assert.ErrorIs(t, ErrInvalidKey, err) } + +func TestNewIndexKey_IfEmptyParam_ReturnPrefix(t *testing.T) { + key := NewCollectionIndexKey("", "") + assert.Equal(t, "/collection/index", key.ToString()) +} + +func TestNewIndexKey_ParamsAreGiven_ReturnFullKey(t *testing.T) { + key := NewCollectionIndexKey("col", "idx") + assert.Equal(t, "/collection/index/col/idx", key.ToString()) +} + +func TestNewIndexKey_InNoCollectionName_ReturnJustPrefix(t *testing.T) { + key := NewCollectionIndexKey("", "idx") + assert.Equal(t, "/collection/index", key.ToString()) +} + +func TestNewIndexKey_InNoIndexName_ReturnWithoutIndexName(t *testing.T) { + key := NewCollectionIndexKey("col", "") + assert.Equal(t, "/collection/index/col", key.ToString()) +} From ec2d76185c353533531cd5c970370204c002be90 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 2 May 2023 19:01:45 +0200 Subject: [PATCH 012/120] Add index key from string --- core/key.go | 12 ++++++++++++ core/key_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/core/key.go b/core/key.go index e800347cf3..7679f8744b 100644 --- a/core/key.go +++ b/core/key.go @@ -222,6 +222,18 @@ func NewCollectionIndexKey(colID, name string) CollectionIndexKey { return CollectionIndexKey{CollectionID: colID, IndexName: name} } +func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { + keyArr := strings.Split(key, "/") + if len(keyArr) < 4 || len(keyArr) > 5 || keyArr[1] != "collection" || keyArr[2] != "index" { + return CollectionIndexKey{}, errors.WithStack(ErrInvalidKey, errors.NewKV("Key", key)) + } + result := CollectionIndexKey{CollectionID: keyArr[3]} + if len(keyArr) == 5 { + result.IndexName = keyArr[4] + } + return result, nil +} + func NewSequenceKey(name string) SequenceKey { return SequenceKey{SequenceName: name} } diff --git a/core/key_test.go b/core/key_test.go index 2e1ad63cad..f23c63e7c5 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -127,3 +127,31 @@ func TestNewIndexKey_InNoIndexName_ReturnWithoutIndexName(t *testing.T) { key := NewCollectionIndexKey("col", "") assert.Equal(t, "/collection/index/col", key.ToString()) } + +func TestNewIndexKeyFromString_IfInvalidString_ReturnError(t *testing.T) { + for _, key := range []string{ + "", + "/collection", + "/collection/index", + "/collection/index/col/idx/extra", + "/wrong/index/col/idx", + "/collection/wrong/col/idx", + } { + _, err := NewCollectionIndexKeyFromString(key) + assert.ErrorIs(t, err, ErrInvalidKey) + } +} + +func TestNewIndexKeyFromString_IfOnlyCollectionName_ReturnKey(t *testing.T) { + key, err := NewCollectionIndexKeyFromString("/collection/index/col") + assert.NoError(t, err) + assert.Equal(t, key.CollectionID, "col") + assert.Equal(t, key.IndexName, "") +} + +func TestNewIndexKeyFromString_IfFullKeyString_ReturnKey(t *testing.T) { + key, err := NewCollectionIndexKeyFromString("/collection/index/col/idx") + assert.NoError(t, err) + assert.Equal(t, key.CollectionID, "col") + assert.Equal(t, key.IndexName, "idx") +} From d2f7f4af32f65f79ca38ab3cf12cda1e499c5a15 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 May 2023 11:16:25 +0200 Subject: [PATCH 013/120] Add retrieving list of indexes --- client/index.go | 5 ++ db/collection.go | 37 ++++++++++++++ db/index_test.go | 130 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 153 insertions(+), 19 deletions(-) diff --git a/client/index.go b/client/index.go index 453033ed9d..a7febb45c9 100644 --- a/client/index.go +++ b/client/index.go @@ -17,3 +17,8 @@ type IndexDescription struct { Fields []IndexedFieldDescription Unique bool } + +type CollectionIndexDescription struct { + CollectionName string + Index IndexDescription +} diff --git a/db/collection.go b/db/collection.go index e4ac4163eb..294ab4d9ee 100644 --- a/db/collection.go +++ b/db/collection.go @@ -201,6 +201,43 @@ func (db *db) createCollectionIndex( return col.CreateIndex(ctx, desc) } +// getAllCollectionIndexes returns all the indexes in the database. +func (db *db) getAllCollectionIndexes( + ctx context.Context, + txn datastore.Txn, +) ([]client.CollectionIndexDescription, error) { + prefix := core.NewCollectionIndexKey("", "") + q, err := txn.Systemstore().Query(ctx, query.Query{ + Prefix: prefix.ToString(), + }) + if err != nil { + return nil, NewErrFailedToCreateCollectionQuery(err) + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + indexes := make([]client.CollectionIndexDescription, 0) + for res := range q.Next() { + if res.Error != nil { + return nil, err + } + + var colDesk client.IndexDescription + err = json.Unmarshal(res.Value, &colDesk) + indexKey, err := core.NewCollectionIndexKeyFromString(res.Key) + err = err + indexes = append(indexes, client.CollectionIndexDescription{ + CollectionName: indexKey.CollectionID, + Index: colDesk, + }) + } + + return indexes, nil +} + // updateCollection updates the persisted collection description matching the name of the given // description, to the values in the given description. // diff --git a/db/index_test.go b/db/index_test.go index 92d5c51cbc..c7b4c420a6 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -16,34 +16,64 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" ) +const ( + usersColName = "Users" + productsColName = "Products" +) + type indexTestFixture struct { ctx context.Context db *implicitTxnDB txn datastore.Txn collection client.Collection + t *testing.T } func (f *indexTestFixture) createCollectionIndex( desc client.IndexDescription, ) (client.IndexDescription, error) { - return f.db.createCollectionIndex(f.ctx, f.txn, f.collection.Name(), desc) + return f.createCollectionIndexFor(f.collection.Name(), desc) } -func newIndexTestFixture(t *testing.T) *indexTestFixture { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - assert.NoError(t, err) +func (f *indexTestFixture) createCollectionIndexFor( + collectionName string, + desc client.IndexDescription, +) (client.IndexDescription, error) { + newDesc, err := f.db.createCollectionIndex(f.ctx, f.txn, collectionName, desc) + //if err != nil { + //return newDesc, err + //} + //f.txn, err = f.db.NewTxn(f.ctx, false) + //assert.NoError(f.t, err) + return newDesc, err +} + +func (f *indexTestFixture) getAllCollectionIndexes() ([]client.CollectionIndexDescription, error) { + return f.db.getAllCollectionIndexes(f.ctx, f.txn) +} + +func (f *indexTestFixture) createCollection( + desc client.CollectionDescription, +) client.Collection { + col, err := f.db.createCollection(f.ctx, f.txn, desc) + assert.NoError(f.t, err) + err = f.txn.Commit(f.ctx) + assert.NoError(f.t, err) + f.txn, err = f.db.NewTxn(f.ctx, false) + assert.NoError(f.t, err) + return col +} - desc := client.CollectionDescription{ - Name: "Users", +func getUsersCollectionDesc() client.CollectionDescription { + return client.CollectionDescription{ + Name: usersColName, Schema: client.SchemaDescription{ Fields: []client.FieldDescription{ { @@ -68,20 +98,47 @@ func newIndexTestFixture(t *testing.T) *indexTestFixture { }, }, } +} - col, err := db.createCollection(ctx, txn, desc) - assert.NoError(t, err) +func getProductsCollectionDesc() client.CollectionDescription { + return client.CollectionDescription{ + Name: productsColName, + Schema: client.SchemaDescription{ + Fields: []client.FieldDescription{ + { + Name: "_key", + Kind: client.FieldKind_DocKey, + }, + { + Name: "price", + Kind: client.FieldKind_FLOAT, + Typ: client.LWW_REGISTER, + }, + { + Name: "description", + Kind: client.FieldKind_STRING, + Typ: client.LWW_REGISTER, + }, + }, + }, + } +} - //err = txn.Commit(ctx) - //assert.NoError(t, err) +func newIndexTestFixture(t *testing.T) *indexTestFixture { + ctx := context.Background() + db, err := newMemoryDB(ctx) + assert.NoError(t, err) + txn, err := db.NewTxn(ctx, false) + assert.NoError(t, err) - return &indexTestFixture{ - ctx: ctx, - db: db, - txn: txn, - collection: col, + f := &indexTestFixture{ + ctx: ctx, + db: db, + txn: txn, + t: t, } - + f.collection = f.createCollection(getUsersCollectionDesc()) + return f } func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { @@ -221,3 +278,38 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { assert.NoError(t, err) assert.Equal(t, deserialized, desc) } + +// test if non-existing property is given for index +// test if collection exists before creating an index for it + +func TestGetIndexes_ShouldReturnListOfExistingIndexes(t *testing.T) { + f := newIndexTestFixture(t) + + usersIndexDesc := client.IndexDescription{ + Name: "users_name_index", + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) + assert.NoError(t, err) + + f.createCollection(getProductsCollectionDesc()) + productsIndexDesc := client.IndexDescription{ + Name: "products_description_index", + Fields: []client.IndexedFieldDescription{{Name: "price"}}, + } + _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) + assert.NoError(t, err) + + indexes, err := f.getAllCollectionIndexes() + assert.NoError(t, err) + + require.Equal(t, len(indexes), 2) + usersIndexIndex := 0 + if indexes[0].CollectionName != usersColName { + usersIndexIndex = 1 + } + assert.Equal(t, indexes[usersIndexIndex].Index, usersIndexDesc) + assert.Equal(t, indexes[usersIndexIndex].CollectionName, usersColName) + assert.Equal(t, indexes[1-usersIndexIndex].Index, productsIndexDesc) + assert.Equal(t, indexes[1-usersIndexIndex].CollectionName, productsColName) +} From 4fdc1212d0e90d8398378d79236e093ad28d7c83 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 May 2023 12:54:40 +0200 Subject: [PATCH 014/120] Add retrieving list of collection indexes --- db/collection.go | 32 ++++++++++++++ db/index_test.go | 107 +++++++++++++++++++++++++++++++---------------- 2 files changed, 102 insertions(+), 37 deletions(-) diff --git a/db/collection.go b/db/collection.go index 294ab4d9ee..75fafd453f 100644 --- a/db/collection.go +++ b/db/collection.go @@ -238,6 +238,38 @@ func (db *db) getAllCollectionIndexes( return indexes, nil } +func (db *db) getCollectionIndexes( + ctx context.Context, + txn datastore.Txn, + colName string, +) ([]client.IndexDescription, error) { + prefix := core.NewCollectionIndexKey(colName, "") + q, err := txn.Systemstore().Query(ctx, query.Query{ + Prefix: prefix.ToString(), + }) + if err != nil { + //return nil, NewErrFailedToCreateCollectionQuery(err) + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + indexes := make([]client.IndexDescription, 0) + for res := range q.Next() { + if res.Error != nil { + return nil, err + } + + var colDesk client.IndexDescription + err = json.Unmarshal(res.Value, &colDesk) + indexes = append(indexes, colDesk) + } + + return indexes, nil +} + // updateCollection updates the persisted collection description matching the name of the given // description, to the values in the given description. // diff --git a/db/index_test.go b/db/index_test.go index c7b4c420a6..c4cbd718a3 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -36,41 +36,6 @@ type indexTestFixture struct { t *testing.T } -func (f *indexTestFixture) createCollectionIndex( - desc client.IndexDescription, -) (client.IndexDescription, error) { - return f.createCollectionIndexFor(f.collection.Name(), desc) -} - -func (f *indexTestFixture) createCollectionIndexFor( - collectionName string, - desc client.IndexDescription, -) (client.IndexDescription, error) { - newDesc, err := f.db.createCollectionIndex(f.ctx, f.txn, collectionName, desc) - //if err != nil { - //return newDesc, err - //} - //f.txn, err = f.db.NewTxn(f.ctx, false) - //assert.NoError(f.t, err) - return newDesc, err -} - -func (f *indexTestFixture) getAllCollectionIndexes() ([]client.CollectionIndexDescription, error) { - return f.db.getAllCollectionIndexes(f.ctx, f.txn) -} - -func (f *indexTestFixture) createCollection( - desc client.CollectionDescription, -) client.Collection { - col, err := f.db.createCollection(f.ctx, f.txn, desc) - assert.NoError(f.t, err) - err = f.txn.Commit(f.ctx) - assert.NoError(f.t, err) - f.txn, err = f.db.NewTxn(f.ctx, false) - assert.NoError(f.t, err) - return col -} - func getUsersCollectionDesc() client.CollectionDescription { return client.CollectionDescription{ Name: usersColName, @@ -141,6 +106,45 @@ func newIndexTestFixture(t *testing.T) *indexTestFixture { return f } +func (f *indexTestFixture) createCollectionIndex( + desc client.IndexDescription, +) (client.IndexDescription, error) { + return f.createCollectionIndexFor(f.collection.Name(), desc) +} + +func (f *indexTestFixture) createCollectionIndexFor( + collectionName string, + desc client.IndexDescription, +) (client.IndexDescription, error) { + newDesc, err := f.db.createCollectionIndex(f.ctx, f.txn, collectionName, desc) + //if err != nil { + //return newDesc, err + //} + //f.txn, err = f.db.NewTxn(f.ctx, false) + //assert.NoError(f.t, err) + return newDesc, err +} + +func (f *indexTestFixture) getAllIndexes() ([]client.CollectionIndexDescription, error) { + return f.db.getAllCollectionIndexes(f.ctx, f.txn) +} + +func (f *indexTestFixture) getCollectionIndexes(colName string) ([]client.IndexDescription, error) { + return f.db.getCollectionIndexes(f.ctx, f.txn, colName) +} + +func (f *indexTestFixture) createCollection( + desc client.CollectionDescription, +) client.Collection { + col, err := f.db.createCollection(f.ctx, f.txn, desc) + assert.NoError(f.t, err) + err = f.txn.Commit(f.ctx) + assert.NoError(f.t, err) + f.txn, err = f.db.NewTxn(f.ctx, false) + assert.NoError(f.t, err) + return col +} + func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { f := newIndexTestFixture(t) @@ -282,7 +286,7 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { // test if non-existing property is given for index // test if collection exists before creating an index for it -func TestGetIndexes_ShouldReturnListOfExistingIndexes(t *testing.T) { +func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { f := newIndexTestFixture(t) usersIndexDesc := client.IndexDescription{ @@ -300,7 +304,7 @@ func TestGetIndexes_ShouldReturnListOfExistingIndexes(t *testing.T) { _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) assert.NoError(t, err) - indexes, err := f.getAllCollectionIndexes() + indexes, err := f.getAllIndexes() assert.NoError(t, err) require.Equal(t, len(indexes), 2) @@ -313,3 +317,32 @@ func TestGetIndexes_ShouldReturnListOfExistingIndexes(t *testing.T) { assert.Equal(t, indexes[1-usersIndexIndex].Index, productsIndexDesc) assert.Equal(t, indexes[1-usersIndexIndex].CollectionName, productsColName) } + +func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) { + f := newIndexTestFixture(t) + + usersIndexDesc := client.IndexDescription{ + Name: "users_name_index", + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) + assert.NoError(t, err) + + f.createCollection(getProductsCollectionDesc()) + productsIndexDesc := client.IndexDescription{ + Name: "products_description_index", + Fields: []client.IndexedFieldDescription{{Name: "price"}}, + } + _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) + assert.NoError(t, err) + + userIndexes, err := f.getCollectionIndexes(usersColName) + assert.NoError(t, err) + require.Equal(t, len(userIndexes), 1) + assert.Equal(t, userIndexes[0], usersIndexDesc) + + productIndexes, err := f.getCollectionIndexes(productsColName) + assert.NoError(t, err) + require.Equal(t, len(productIndexes), 1) + assert.Equal(t, productIndexes[0], productsIndexDesc) +} From dcf11c306db14bdbf779d1b75d99cb3fffcb0c4b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 May 2023 13:16:44 +0200 Subject: [PATCH 015/120] Add storage failure tests --- db/collection.go | 7 +++++-- db/index_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/db/collection.go b/db/collection.go index 75fafd453f..61b56b56dd 100644 --- a/db/collection.go +++ b/db/collection.go @@ -196,7 +196,10 @@ func (db *db) createCollectionIndex( collectionName string, desc client.IndexDescription, ) (client.IndexDescription, error) { - col, _ := db.getCollectionByName(ctx, txn, collectionName) // TODO: test error + col, err := db.getCollectionByName(ctx, txn, collectionName) // TODO: test error + if err != nil { + return client.IndexDescription{}, err + } col = col.WithTxn(txn) return col.CreateIndex(ctx, desc) } @@ -248,7 +251,7 @@ func (db *db) getCollectionIndexes( Prefix: prefix.ToString(), }) if err != nil { - //return nil, NewErrFailedToCreateCollectionQuery(err) + return nil, NewErrFailedToCreateCollectionQuery(err) } defer func() { if err := q.Close(); err != nil { diff --git a/db/index_test.go b/db/index_test.go index c4cbd718a3..9a8b3c5dc6 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -283,6 +283,21 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { assert.Equal(t, deserialized, desc) } +func TestCreateIndex_IfStorageFails_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + name := "users_age_ASC" + desc := client.IndexDescription{ + Name: name, + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + + f.db.Close(f.ctx) + + _, err := f.createCollectionIndex(desc) + assert.Error(t, err) +} + // test if non-existing property is given for index // test if collection exists before creating an index for it @@ -346,3 +361,19 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) require.Equal(t, len(productIndexes), 1) assert.Equal(t, productIndexes[0], productsIndexDesc) } + +func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + usersIndexDesc := client.IndexDescription{ + Name: "users_name_index", + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + } + _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) + assert.NoError(t, err) + + f.db.Close(f.ctx) + + _, err = f.getCollectionIndexes(usersColName) + assert.Error(t, err) +} From 0d848ab309a86c783653ae5ab3623a1097fc4745 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 May 2023 15:27:55 +0200 Subject: [PATCH 016/120] Check if stored index is valid --- db/collection.go | 3 +++ db/errors.go | 9 +++++++++ db/index_test.go | 11 +++++++++++ 3 files changed, 23 insertions(+) diff --git a/db/collection.go b/db/collection.go index 61b56b56dd..5e0cab3145 100644 --- a/db/collection.go +++ b/db/collection.go @@ -267,6 +267,9 @@ func (db *db) getCollectionIndexes( var colDesk client.IndexDescription err = json.Unmarshal(res.Value, &colDesk) + if err != nil { + return nil, NewErrInvalidStoredIndex(err) + } indexes = append(indexes, colDesk) } diff --git a/db/errors.go b/db/errors.go index 34b8ed335d..c49addbdc8 100644 --- a/db/errors.go +++ b/db/errors.go @@ -42,6 +42,8 @@ const ( errIndexFieldMissingDirection string = "index field missing direction" errIndexSingleFieldWrongDirection string = "wrong direction for index with a single field" errIndexWithNameAlreadyExists string = "index with name already exists" + errCollectionDoesntExist string = "collection with name doesn't exist" + errInvalidStoredIndex string = "invalid stored index" ) var ( @@ -94,6 +96,7 @@ var ( ErrIndexFieldMissingDirection = errors.New(errIndexFieldMissingDirection) ErrIndexSingleFieldWrongDirection = errors.New(errIndexSingleFieldWrongDirection) ErrIndexWithNameAlreadyExists = errors.New(errIndexWithNameAlreadyExists) + ErrCollectionDoesntExist = errors.New(errCollectionDoesntExist) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document @@ -108,6 +111,12 @@ func NewErrFailedToCreateCollectionQuery(inner error) error { return errors.Wrap(errFailedToCreateCollectionQuery, inner) } +// NewErrInvalidStoredIndex returns a new error indicating that the stored +// index in the database is invalid. +func NewErrInvalidStoredIndex(inner error) error { + return errors.Wrap(errInvalidStoredIndex, inner) +} + // NewErrFailedToGetCollection returns a new error indicating that the collection could not be obtained. func NewErrFailedToGetCollection(name string, inner error) error { return errors.Wrap(errFailedToGetCollection, inner, errors.NewKV("Name", name)) diff --git a/db/index_test.go b/db/index_test.go index 9a8b3c5dc6..703a25862b 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -377,3 +377,14 @@ func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { _, err = f.getCollectionIndexes(usersColName) assert.Error(t, err) } + +func TestGetCollectionIndexes_InvalidIndexIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) + assert.NoError(t, err) + + _, err = f.getCollectionIndexes(usersColName) + assert.Error(t, err) +} From a9b0639a4c2417324767da251fb3430c92fcab1c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 May 2023 17:15:57 +0200 Subject: [PATCH 017/120] Check if indexes property exists --- db/errors.go | 9 +++++++-- db/index.go | 26 ++++++++++++++++++++++++++ db/index_test.go | 29 +++++++++++++++++++++++++---- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/db/errors.go b/db/errors.go index c49addbdc8..d75b04825f 100644 --- a/db/errors.go +++ b/db/errors.go @@ -42,8 +42,8 @@ const ( errIndexFieldMissingDirection string = "index field missing direction" errIndexSingleFieldWrongDirection string = "wrong direction for index with a single field" errIndexWithNameAlreadyExists string = "index with name already exists" - errCollectionDoesntExist string = "collection with name doesn't exist" errInvalidStoredIndex string = "invalid stored index" + errNonExistingFieldForIndex string = "creating an index on a non-existing property" ) var ( @@ -96,7 +96,6 @@ var ( ErrIndexFieldMissingDirection = errors.New(errIndexFieldMissingDirection) ErrIndexSingleFieldWrongDirection = errors.New(errIndexSingleFieldWrongDirection) ErrIndexWithNameAlreadyExists = errors.New(errIndexWithNameAlreadyExists) - ErrCollectionDoesntExist = errors.New(errCollectionDoesntExist) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document @@ -117,6 +116,12 @@ func NewErrInvalidStoredIndex(inner error) error { return errors.Wrap(errInvalidStoredIndex, inner) } +// NewErrNonExistingFieldForIndex returns a new error indicating the attempt to create an index +// on a non-existing property. +func NewErrNonExistingFieldForIndex(prop string) error { + return errors.New(errNonExistingFieldForIndex, errors.NewKV("Property", prop)) +} + // NewErrFailedToGetCollection returns a new error indicating that the collection could not be obtained. func NewErrFailedToGetCollection(name string, inner error) error { return errors.Wrap(errFailedToGetCollection, inner, errors.NewKV("Name", name)) diff --git a/db/index.go b/db/index.go index 84f829bba5..6ad0c32803 100644 --- a/db/index.go +++ b/db/index.go @@ -104,6 +104,11 @@ func (c *collection) createIndex( return nil, err } + err = c.checkExistingFields(ctx, desc.Fields) + if err != nil { + return nil, err + } + indexKey, err := c.processIndexName(ctx, &desc) if err != nil { return nil, err @@ -127,6 +132,27 @@ func (c *collection) createIndex( return colIndex, nil } +func (c *collection) checkExistingFields( + ctx context.Context, + fields []client.IndexedFieldDescription, +) error { + collectionFields := c.Description().Schema.Fields + for _, field := range fields { + found := false + fieldLower := strings.ToLower(field.Name) + for _, colField := range collectionFields { + if fieldLower == strings.ToLower(colField.Name) { + found = true + break + } + } + if !found { + return NewErrNonExistingFieldForIndex(field.Name) + } + } + return nil +} + func (c *collection) processIndexName( ctx context.Context, desc *client.IndexDescription, diff --git a/db/index_test.go b/db/index_test.go index 703a25862b..1ad523ea2f 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -204,7 +204,8 @@ func TestCreateIndex_IfNameIsNotSpecified_GenerateWithLowerCase(t *testing.T) { }, } f.collection.Description().Schema.Fields[1].Name = "Name" - newDesc, _ := f.createCollectionIndex(desc) + newDesc, err := f.createCollectionIndex(desc) + assert.NoError(t, err) assert.Equal(t, newDesc.Name, "users_name_ASC") } @@ -298,8 +299,28 @@ func TestCreateIndex_IfStorageFails_ReturnError(t *testing.T) { assert.Error(t, err) } -// test if non-existing property is given for index -// test if collection exists before creating an index for it +func TestCreateIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + desc := client.IndexDescription{ + Fields: []client.IndexedFieldDescription{{Name: "price"}}, + } + + _, err := f.createCollectionIndexFor(productsColName, desc) + assert.Error(t, err) +} + +func TestCreateIndex_IfPropertyDoesntExist_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + const prop = "non_existing_property" + desc := client.IndexDescription{ + Fields: []client.IndexedFieldDescription{{Name: prop}}, + } + + _, err := f.createCollectionIndex(desc) + assert.ErrorIs(t, err, NewErrNonExistingFieldForIndex(prop)) +} func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { f := newIndexTestFixture(t) @@ -386,5 +407,5 @@ func TestGetCollectionIndexes_InvalidIndexIsStored_ReturnError(t *testing.T) { assert.NoError(t, err) _, err = f.getCollectionIndexes(usersColName) - assert.Error(t, err) + assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) } From 1035215374e19bf55d070e0ad3ece7689f5e575f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 May 2023 18:21:26 +0200 Subject: [PATCH 018/120] Check storage errors --- db/collection.go | 7 ++++++- db/errors.go | 15 +++++++++++---- db/index_test.go | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/db/collection.go b/db/collection.go index 5e0cab3145..0e483a5d7e 100644 --- a/db/collection.go +++ b/db/collection.go @@ -230,8 +230,13 @@ func (db *db) getAllCollectionIndexes( var colDesk client.IndexDescription err = json.Unmarshal(res.Value, &colDesk) + if err != nil { + return nil, NewErrInvalidStoredIndex(err) + } indexKey, err := core.NewCollectionIndexKeyFromString(res.Key) - err = err + if err != nil { + return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) + } indexes = append(indexes, client.CollectionIndexDescription{ CollectionName: indexKey.CollectionID, Index: colDesk, diff --git a/db/errors.go b/db/errors.go index d75b04825f..362a0beb26 100644 --- a/db/errors.go +++ b/db/errors.go @@ -43,7 +43,8 @@ const ( errIndexSingleFieldWrongDirection string = "wrong direction for index with a single field" errIndexWithNameAlreadyExists string = "index with name already exists" errInvalidStoredIndex string = "invalid stored index" - errNonExistingFieldForIndex string = "creating an index on a non-existing property" + errInvalidStoredIndexKey string = "invalid stored index key" + errNonExistingFieldForIndex string = "creating an index on a non-existing property" ) var ( @@ -116,10 +117,16 @@ func NewErrInvalidStoredIndex(inner error) error { return errors.Wrap(errInvalidStoredIndex, inner) } +// NewErrInvalidStoredIndexKey returns a new error indicating that the stored +// index in the database is invalid. +func NewErrInvalidStoredIndexKey(key string) error { + return errors.New(errInvalidStoredIndexKey, errors.NewKV("Key", key)) +} + // NewErrNonExistingFieldForIndex returns a new error indicating the attempt to create an index -// on a non-existing property. -func NewErrNonExistingFieldForIndex(prop string) error { - return errors.New(errNonExistingFieldForIndex, errors.NewKV("Property", prop)) +// on a non-existing field. +func NewErrNonExistingFieldForIndex(field string) error { + return errors.New(errNonExistingFieldForIndex, errors.NewKV("Field", field)) } // NewErrFailedToGetCollection returns a new error indicating that the collection could not be obtained. diff --git a/db/index_test.go b/db/index_test.go index 1ad523ea2f..899875b27d 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -15,6 +15,7 @@ import ( "encoding/json" "testing" + ds "github.com/ipfs/go-datastore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -399,7 +400,7 @@ func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { assert.Error(t, err) } -func TestGetCollectionIndexes_InvalidIndexIsStored_ReturnError(t *testing.T) { +func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { f := newIndexTestFixture(t) indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") @@ -409,3 +410,33 @@ func TestGetCollectionIndexes_InvalidIndexIsStored_ReturnError(t *testing.T) { _, err = f.getCollectionIndexes(usersColName) assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) } + +func TestGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) + assert.NoError(t, err) + + _, err = f.getAllIndexes() + assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) +} + +func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + key := ds.NewKey(indexKey.ToString() + "/invalid") + desc := client.IndexDescription{ + Name: "some_index_name", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + } + descData, _ := json.Marshal(desc) + err := f.txn.Systemstore().Put(f.ctx, key, descData) + assert.NoError(t, err) + + _, err = f.getAllIndexes() + assert.ErrorIs(t, err, NewErrInvalidStoredIndexKey(key.String())) +} From 0d75f5802c8ad0943825ec6001eaa7a8dcede2f8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 4 May 2023 12:21:55 +0200 Subject: [PATCH 019/120] Add drop index --- db/collection.go | 17 ++++++++++++++-- db/errors.go | 6 ++++++ db/index.go | 8 +++++++- db/index_test.go | 53 +++++++++++++++++++++++++++++++++++++----------- 4 files changed, 69 insertions(+), 15 deletions(-) diff --git a/db/collection.go b/db/collection.go index 0e483a5d7e..877430b4a5 100644 --- a/db/collection.go +++ b/db/collection.go @@ -196,14 +196,27 @@ func (db *db) createCollectionIndex( collectionName string, desc client.IndexDescription, ) (client.IndexDescription, error) { - col, err := db.getCollectionByName(ctx, txn, collectionName) // TODO: test error + col, err := db.getCollectionByName(ctx, txn, collectionName) if err != nil { - return client.IndexDescription{}, err + return client.IndexDescription{}, NewErrCollectionDoesntExist(collectionName) } col = col.WithTxn(txn) return col.CreateIndex(ctx, desc) } +func (db *db) dropCollectionIndex( + ctx context.Context, + txn datastore.Txn, + collectionName, indexName string, +) error { + col, err := db.getCollectionByName(ctx, txn, collectionName) + if err != nil { + return NewErrCollectionDoesntExist(collectionName) + } + col = col.WithTxn(txn) + return col.DropIndex(ctx, indexName) +} + // getAllCollectionIndexes returns all the indexes in the database. func (db *db) getAllCollectionIndexes( ctx context.Context, diff --git a/db/errors.go b/db/errors.go index 362a0beb26..ee28aeeb88 100644 --- a/db/errors.go +++ b/db/errors.go @@ -45,6 +45,7 @@ const ( errInvalidStoredIndex string = "invalid stored index" errInvalidStoredIndexKey string = "invalid stored index key" errNonExistingFieldForIndex string = "creating an index on a non-existing property" + errCollectionDoesntExisting string = "collection with given name doesn't exist" ) var ( @@ -129,6 +130,11 @@ func NewErrNonExistingFieldForIndex(field string) error { return errors.New(errNonExistingFieldForIndex, errors.NewKV("Field", field)) } +// NewErrCollectionDoesntExist returns a new error indicating the collection doesn't exist. +func NewErrCollectionDoesntExist(colName string) error { + return errors.New(errCollectionDoesntExisting, errors.NewKV("Collection", colName)) +} + // NewErrFailedToGetCollection returns a new error indicating that the collection could not be obtained. func NewErrFailedToGetCollection(name string, inner error) error { return errors.Wrap(errFailedToGetCollection, inner, errors.NewKV("Name", name)) diff --git a/db/index.go b/db/index.go index 6ad0c32803..c66cb03e53 100644 --- a/db/index.go +++ b/db/index.go @@ -88,7 +88,13 @@ func (c *collection) CreateIndex( } func (c *collection) DropIndex(ctx context.Context, indexName string) error { - return nil + key := core.NewCollectionIndexKey(c.Name(), indexName) + + txn, err := c.getTxn(ctx, false) + if err != nil { + return err + } + return txn.Systemstore().Delete(ctx, key.ToDS()) } func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { diff --git a/db/index_test.go b/db/index_test.go index 899875b27d..4b86f9c15a 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -113,6 +113,22 @@ func (f *indexTestFixture) createCollectionIndex( return f.createCollectionIndexFor(f.collection.Name(), desc) } +func (f *indexTestFixture) createUserCollectionIndex() client.IndexDescription { + desc := client.IndexDescription{ + Name: "some_index_name", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + } + newDesc, err := f.createCollectionIndexFor(f.collection.Name(), desc) + assert.NoError(f.t, err) + return newDesc +} + +func (f *indexTestFixture) dropIndex(colName, indexName string) error { + return f.db.dropCollectionIndex(f.ctx, f.txn, colName, indexName) +} + func (f *indexTestFixture) createCollectionIndexFor( collectionName string, desc client.IndexDescription, @@ -308,19 +324,19 @@ func TestCreateIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { } _, err := f.createCollectionIndexFor(productsColName, desc) - assert.Error(t, err) + assert.ErrorIs(t, err, NewErrCollectionDoesntExist(usersColName)) } func TestCreateIndex_IfPropertyDoesntExist_ReturnError(t *testing.T) { f := newIndexTestFixture(t) - const prop = "non_existing_property" + const field = "non_existing_field" desc := client.IndexDescription{ - Fields: []client.IndexedFieldDescription{{Name: prop}}, + Fields: []client.IndexedFieldDescription{{Name: field}}, } _, err := f.createCollectionIndex(desc) - assert.ErrorIs(t, err, NewErrNonExistingFieldForIndex(prop)) + assert.ErrorIs(t, err, NewErrNonExistingFieldForIndex(field)) } func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { @@ -386,17 +402,11 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) - - usersIndexDesc := client.IndexDescription{ - Name: "users_name_index", - Fields: []client.IndexedFieldDescription{{Name: "name"}}, - } - _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) - assert.NoError(t, err) + f.createUserCollectionIndex() f.db.Close(f.ctx) - _, err = f.getCollectionIndexes(usersColName) + _, err := f.getCollectionIndexes(usersColName) assert.Error(t, err) } @@ -440,3 +450,22 @@ func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { _, err = f.getAllIndexes() assert.ErrorIs(t, err, NewErrInvalidStoredIndexKey(key.String())) } + +func TestDropIndex_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + desc := f.createUserCollectionIndex() + + err := f.dropIndex(usersColName, desc.Name) + assert.NoError(t, err) + + indexKey := core.NewCollectionIndexKey(usersColName, desc.Name) + _, err = f.txn.Systemstore().Get(f.ctx, indexKey.ToDS()) + assert.Error(t, err) +} + +func TestDropIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + err := f.dropIndex(productsColName, "any_name") + assert.ErrorIs(t, err, NewErrCollectionDoesntExist(usersColName)) +} From ac7a6f59a953e57c29f923f669fa5055b5ed54fe Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 4 May 2023 17:51:06 +0200 Subject: [PATCH 020/120] Add drop all indexes --- client/collection.go | 3 ++ db/collection.go | 13 +++++ db/index.go | 33 ++++++++++++ db/index_test.go | 124 +++++++++++++++++++++++++++++++++---------- 4 files changed, 146 insertions(+), 27 deletions(-) diff --git a/client/collection.go b/client/collection.go index c1f5e821cf..b5f7c64eb6 100644 --- a/client/collection.go +++ b/client/collection.go @@ -143,6 +143,9 @@ type Collection interface { // DropIndex drops an index from the collection. DropIndex(ctx context.Context, indexName string) error + // DropAllIndexes drops all indexes from the collection. + DropAllIndexes(ctx context.Context) error + // GetIndexes returns all the indexes that exist on the collection. GetIndexes(ctx context.Context) ([]IndexDescription, error) } diff --git a/db/collection.go b/db/collection.go index 877430b4a5..4f8c7d6e93 100644 --- a/db/collection.go +++ b/db/collection.go @@ -217,6 +217,19 @@ func (db *db) dropCollectionIndex( return col.DropIndex(ctx, indexName) } +func (db *db) dropAllCollectionIndex( + ctx context.Context, + txn datastore.Txn, + collectionName string, +) error { + col, err := db.getCollectionByName(ctx, txn, collectionName) + if err != nil { + return NewErrCollectionDoesntExist(collectionName) + } + col = col.WithTxn(txn) + return col.DropAllIndexes(ctx) +} + // getAllCollectionIndexes returns all the indexes in the database. func (db *db) getAllCollectionIndexes( ctx context.Context, diff --git a/db/index.go b/db/index.go index c66cb03e53..3c8a0caa5a 100644 --- a/db/index.go +++ b/db/index.go @@ -6,6 +6,9 @@ import ( "strconv" "strings" + ds "github.com/ipfs/go-datastore" + + "github.com/ipfs/go-datastore/query" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" ) @@ -97,6 +100,36 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { return txn.Systemstore().Delete(ctx, key.ToDS()) } +func (c *collection) DropAllIndexes(ctx context.Context) error { + prefix := core.NewCollectionIndexKey(c.Name(), "") + txn, err := c.getTxn(ctx, false) + if err != nil { + return err + } + q, err := txn.Systemstore().Query(ctx, query.Query{ + Prefix: prefix.ToString(), + }) + if err != nil { + return err + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + for res := range q.Next() { + if res.Error != nil { + return res.Error + } + err = txn.Systemstore().Delete(ctx, ds.NewKey(res.Key)) + if err != nil { + return err + } + } + return nil +} + func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { return nil, nil } diff --git a/db/index_test.go b/db/index_test.go index 4b86f9c15a..b7bc61fa61 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -16,6 +16,7 @@ import ( "testing" ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -129,6 +130,31 @@ func (f *indexTestFixture) dropIndex(colName, indexName string) error { return f.db.dropCollectionIndex(f.ctx, f.txn, colName, indexName) } +func (f *indexTestFixture) dropAllIndexes(colName string) error { + return f.db.dropAllCollectionIndex(f.ctx, f.txn, colName) +} + +func (f *indexTestFixture) countIndexPrefixes(colName, indexName string) int { + prefix := core.NewCollectionIndexKey(usersColName, indexName) + q, err := f.txn.Systemstore().Query(f.ctx, query.Query{ + Prefix: prefix.ToString(), + }) + assert.NoError(f.t, err) + defer func() { + err := q.Close() + assert.NoError(f.t, err) + }() + + count := 0 + for res := range q.Next() { + if res.Error != nil { + assert.NoError(f.t, err) + } + count++ + } + return count +} + func (f *indexTestFixture) createCollectionIndexFor( collectionName string, desc client.IndexDescription, @@ -371,6 +397,36 @@ func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { assert.Equal(t, indexes[1-usersIndexIndex].CollectionName, productsColName) } +func TestGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) + assert.NoError(t, err) + + _, err = f.getAllIndexes() + assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) +} + +func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + key := ds.NewKey(indexKey.ToString() + "/invalid") + desc := client.IndexDescription{ + Name: "some_index_name", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Direction: client.Ascending}, + }, + } + descData, _ := json.Marshal(desc) + err := f.txn.Systemstore().Put(f.ctx, key, descData) + assert.NoError(t, err) + + _, err = f.getAllIndexes() + assert.ErrorIs(t, err, NewErrInvalidStoredIndexKey(key.String())) +} + func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) { f := newIndexTestFixture(t) @@ -421,51 +477,65 @@ func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) } -func TestGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { +func TestDropIndex_ShouldDeleteIndex(t *testing.T) { f := newIndexTestFixture(t) + desc := f.createUserCollectionIndex() - indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") - err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) + err := f.dropIndex(usersColName, desc.Name) assert.NoError(t, err) - _, err = f.getAllIndexes() - assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) + indexKey := core.NewCollectionIndexKey(usersColName, desc.Name) + _, err = f.txn.Systemstore().Get(f.ctx, indexKey.ToDS()) + assert.Error(t, err) } -func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { +func TestDropIndex_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) + desc := f.createUserCollectionIndex() - indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") - key := ds.NewKey(indexKey.ToString() + "/invalid") - desc := client.IndexDescription{ - Name: "some_index_name", + f.db.Close(f.ctx) + + err := f.dropIndex(productsColName, desc.Name) + assert.Error(t, err) +} + +func TestDropIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + err := f.dropIndex(productsColName, "any_name") + assert.ErrorIs(t, err, NewErrCollectionDoesntExist(usersColName)) +} + +func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { + f := newIndexTestFixture(t) + _, err := f.createCollectionIndexFor(usersColName, client.IndexDescription{ Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - } - descData, _ := json.Marshal(desc) - err := f.txn.Systemstore().Put(f.ctx, key, descData) - assert.NoError(t, err) + }) + assert.NoError(f.t, err) - _, err = f.getAllIndexes() - assert.ErrorIs(t, err, NewErrInvalidStoredIndexKey(key.String())) -} + _, err = f.createCollectionIndexFor(usersColName, client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + {Name: "age", Direction: client.Ascending}, + }, + }) + assert.NoError(f.t, err) -func TestDropIndex_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) - desc := f.createUserCollectionIndex() + assert.Equal(t, f.countIndexPrefixes(usersColName, ""), 2) - err := f.dropIndex(usersColName, desc.Name) + err = f.dropAllIndexes(usersColName) assert.NoError(t, err) - indexKey := core.NewCollectionIndexKey(usersColName, desc.Name) - _, err = f.txn.Systemstore().Get(f.ctx, indexKey.ToDS()) - assert.Error(t, err) + assert.Equal(t, f.countIndexPrefixes(usersColName, ""), 0) } -func TestDropIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { +func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) + f.createUserCollectionIndex() - err := f.dropIndex(productsColName, "any_name") - assert.ErrorIs(t, err, NewErrCollectionDoesntExist(usersColName)) + f.db.Close(f.ctx) + + err := f.dropAllIndexes(usersColName) + assert.Error(t, err) } From 6e7769cbcb09cbdb5b7bcd708f18055ca66fc3f2 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 5 May 2023 10:57:19 +0200 Subject: [PATCH 021/120] Remove DropAllIndexes from public interface --- client/collection.go | 3 --- db/collection.go | 13 ------------- db/index.go | 2 +- db/index_test.go | 3 ++- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/client/collection.go b/client/collection.go index b5f7c64eb6..c1f5e821cf 100644 --- a/client/collection.go +++ b/client/collection.go @@ -143,9 +143,6 @@ type Collection interface { // DropIndex drops an index from the collection. DropIndex(ctx context.Context, indexName string) error - // DropAllIndexes drops all indexes from the collection. - DropAllIndexes(ctx context.Context) error - // GetIndexes returns all the indexes that exist on the collection. GetIndexes(ctx context.Context) ([]IndexDescription, error) } diff --git a/db/collection.go b/db/collection.go index 4f8c7d6e93..877430b4a5 100644 --- a/db/collection.go +++ b/db/collection.go @@ -217,19 +217,6 @@ func (db *db) dropCollectionIndex( return col.DropIndex(ctx, indexName) } -func (db *db) dropAllCollectionIndex( - ctx context.Context, - txn datastore.Txn, - collectionName string, -) error { - col, err := db.getCollectionByName(ctx, txn, collectionName) - if err != nil { - return NewErrCollectionDoesntExist(collectionName) - } - col = col.WithTxn(txn) - return col.DropAllIndexes(ctx) -} - // getAllCollectionIndexes returns all the indexes in the database. func (db *db) getAllCollectionIndexes( ctx context.Context, diff --git a/db/index.go b/db/index.go index 3c8a0caa5a..072f2dd60a 100644 --- a/db/index.go +++ b/db/index.go @@ -100,7 +100,7 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { return txn.Systemstore().Delete(ctx, key.ToDS()) } -func (c *collection) DropAllIndexes(ctx context.Context) error { +func (c *collection) dropAllIndexes(ctx context.Context) error { prefix := core.NewCollectionIndexKey(c.Name(), "") txn, err := c.getTxn(ctx, false) if err != nil { diff --git a/db/index_test.go b/db/index_test.go index b7bc61fa61..73b657e9cc 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -131,7 +131,8 @@ func (f *indexTestFixture) dropIndex(colName, indexName string) error { } func (f *indexTestFixture) dropAllIndexes(colName string) error { - return f.db.dropAllCollectionIndex(f.ctx, f.txn, colName) + col := (f.collection.WithTxn(f.txn)).(*collection) + return col.dropAllIndexes(f.ctx) } func (f *indexTestFixture) countIndexPrefixes(colName, indexName string) int { From 819da19e1560ff9a6c91f67dec7fddc9a9fc6b5c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 5 May 2023 11:12:20 +0200 Subject: [PATCH 022/120] Return error instead of stack --- core/key.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/key.go b/core/key.go index 7679f8744b..652db2edfd 100644 --- a/core/key.go +++ b/core/key.go @@ -225,7 +225,7 @@ func NewCollectionIndexKey(colID, name string) CollectionIndexKey { func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { keyArr := strings.Split(key, "/") if len(keyArr) < 4 || len(keyArr) > 5 || keyArr[1] != "collection" || keyArr[2] != "index" { - return CollectionIndexKey{}, errors.WithStack(ErrInvalidKey, errors.NewKV("Key", key)) + return CollectionIndexKey{}, ErrInvalidKey } result := CollectionIndexKey{CollectionID: keyArr[3]} if len(keyArr) == 5 { From 020e86a631d54095a55928efe23005181e2308ca Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 5 May 2023 11:56:49 +0200 Subject: [PATCH 023/120] Rename CollectionID to CollectionName --- core/key.go | 12 ++++++------ core/key_test.go | 4 ++-- db/collection.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/key.go b/core/key.go index 652db2edfd..1713b86e3c 100644 --- a/core/key.go +++ b/core/key.go @@ -108,8 +108,8 @@ type CollectionSchemaVersionKey struct { var _ Key = (*CollectionSchemaVersionKey)(nil) type CollectionIndexKey struct { - CollectionID string - IndexName string + CollectionName string + IndexName string } var _ Key = (*CollectionIndexKey)(nil) @@ -219,7 +219,7 @@ func NewCollectionSchemaVersionKey(schemaVersionId string) CollectionSchemaVersi } func NewCollectionIndexKey(colID, name string) CollectionIndexKey { - return CollectionIndexKey{CollectionID: colID, IndexName: name} + return CollectionIndexKey{CollectionName: colID, IndexName: name} } func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { @@ -227,7 +227,7 @@ func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { if len(keyArr) < 4 || len(keyArr) > 5 || keyArr[1] != "collection" || keyArr[2] != "index" { return CollectionIndexKey{}, ErrInvalidKey } - result := CollectionIndexKey{CollectionID: keyArr[3]} + result := CollectionIndexKey{CollectionName: keyArr[3]} if len(keyArr) == 5 { result.IndexName = keyArr[4] } @@ -428,8 +428,8 @@ func (k CollectionSchemaVersionKey) ToDS() ds.Key { func (k CollectionIndexKey) ToString() string { result := COLLECTION_INDEX - if k.CollectionID != "" { - result = result + "/" + k.CollectionID + if k.CollectionName != "" { + result = result + "/" + k.CollectionName if k.IndexName != "" { result = result + "/" + k.IndexName } diff --git a/core/key_test.go b/core/key_test.go index f23c63e7c5..3f09efe44d 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -145,13 +145,13 @@ func TestNewIndexKeyFromString_IfInvalidString_ReturnError(t *testing.T) { func TestNewIndexKeyFromString_IfOnlyCollectionName_ReturnKey(t *testing.T) { key, err := NewCollectionIndexKeyFromString("/collection/index/col") assert.NoError(t, err) - assert.Equal(t, key.CollectionID, "col") + assert.Equal(t, key.CollectionName, "col") assert.Equal(t, key.IndexName, "") } func TestNewIndexKeyFromString_IfFullKeyString_ReturnKey(t *testing.T) { key, err := NewCollectionIndexKeyFromString("/collection/index/col/idx") assert.NoError(t, err) - assert.Equal(t, key.CollectionID, "col") + assert.Equal(t, key.CollectionName, "col") assert.Equal(t, key.IndexName, "idx") } diff --git a/db/collection.go b/db/collection.go index 877430b4a5..5852dfe210 100644 --- a/db/collection.go +++ b/db/collection.go @@ -251,7 +251,7 @@ func (db *db) getAllCollectionIndexes( return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) } indexes = append(indexes, client.CollectionIndexDescription{ - CollectionName: indexKey.CollectionID, + CollectionName: indexKey.CollectionName, Index: colDesk, }) } From b02eb0a520c21c0c71cadb4402f759aba3f126fd Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 5 May 2023 17:29:08 +0200 Subject: [PATCH 024/120] Add custom asserter to test framework --- tests/integration/test_case.go | 23 +++++++++++++++++++++-- tests/integration/utils2.go | 9 +++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 9a8f2300df..0bb1bb0687 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -11,6 +11,8 @@ package tests import ( + "testing" + "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/config" @@ -151,6 +153,20 @@ type UpdateDoc struct { DontSync bool } +// ResultAsserter is an interface that can be implemented to provide custom result +// assertions. +type ResultAsserter interface { + // Assert will be called with the test and the result of the request. + Assert(t *testing.T, result []map[string]any) +} + +// ResultAsserterFunc is a function that can be used to implement the ResultAsserter +type ResultAsserterFunc func(*testing.T, []map[string]any) (bool, string) + +func (f ResultAsserterFunc) Assert(t *testing.T, result []map[string]any) { + f(t, result) +} + // Request represents a standard Defra (GQL) request. type Request struct { // NodeID may hold the ID (index) of a node to execute this request on. @@ -165,6 +181,9 @@ type Request struct { // The expected (data) results of the issued request. Results []map[string]any + // Asserter is an optional custom result asserter. + Asserter ResultAsserter + // Any error expected from the action. Optional. // // String can be a partial, and the test will pass if an error is returned that @@ -176,8 +195,8 @@ type Request struct { // // A new transaction will be created for the first TransactionRequest2 of any given // TransactionId. TransactionRequest2s will be submitted to the database in the order -// in which they are recieved (interleaving amongst other actions if provided), however -// they will not be commited until a TransactionCommit of matching TransactionId is +// in which they are received (interleaving amongst other actions if provided), however +// they will not be committed until a TransactionCommit of matching TransactionId is // provided. type TransactionRequest2 struct { // Used to identify the transaction for this to run against. diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 1f55bb439d..e86041687c 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1014,6 +1014,7 @@ func executeTransactionRequest( &result.GQL, action.Results, action.ExpectedError, + nil, // anyof is not yet supported by transactional requests 0, map[docFieldKey][]any{}, @@ -1072,6 +1073,7 @@ func executeRequest( &result.GQL, action.Results, action.ExpectedError, + action.Asserter, nodeID, anyOfByFieldKey, ) @@ -1142,6 +1144,7 @@ func executeSubscriptionRequest( finalResult, action.Results, action.ExpectedError, + nil, // anyof is not yet supported by subscription requests 0, map[docFieldKey][]any{}, @@ -1215,6 +1218,7 @@ func assertRequestResults( result *client.GQLResult, expectedResults []map[string]any, expectedError string, + asserter ResultAsserter, nodeID int, anyOfByField map[docFieldKey][]any, ) bool { @@ -1229,6 +1233,11 @@ func assertRequestResults( // Note: if result.Data == nil this panics (the panic seems useful while testing). resultantData := result.Data.([]map[string]any) + if asserter != nil { + asserter.Assert(t, resultantData) + return true + } + log.Info(ctx, "", logging.NewKV("RequestResults", result.Data)) // compare results From fdbffe599a198e869e03eafaa199e52d4002ca77 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 May 2023 10:24:25 +0200 Subject: [PATCH 025/120] Prepare fixtures for index testing --- tests/integration/schema/index/simple_test.go | 43 ++++++++ tests/integration/schema/index/utils.go | 103 ++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 tests/integration/schema/index/simple_test.go create mode 100644 tests/integration/schema/index/utils.go diff --git a/tests/integration/schema/index/simple_test.go b/tests/integration/schema/index/simple_test.go new file mode 100644 index 0000000000..78219813aa --- /dev/null +++ b/tests/integration/schema/index/simple_test.go @@ -0,0 +1,43 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestSchemaWithIndexOnTheOnlyField(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type users { + name: String @index + } + `, + }, + createUserDocs(), + testUtils.Request{ + Request: ` + query @explain(type: execute) { + users(filter: {name: {_eq: "Shahzad"}}) { + name + } + }`, + Asserter: newExplainAsserter(2, 8, 1), + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"users"}, test) +} diff --git a/tests/integration/schema/index/utils.go b/tests/integration/schema/index/utils.go new file mode 100644 index 0000000000..ca4dc3981a --- /dev/null +++ b/tests/integration/schema/index/utils.go @@ -0,0 +1,103 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type dataMap = map[string]any + +type explainResultAsserter struct { + iterations int + docFetches int + filterMatches int +} + +func (a explainResultAsserter) Assert(t *testing.T, result []dataMap) { + require.Len(t, result, 1, "Expected one result, got %d", len(result)) + explainNode, ok := result[0]["explain"].(dataMap) + require.True(t, ok, "Expected explain") + assert.Equal(t, explainNode["executionSuccess"], true) + assert.Equal(t, explainNode["sizeOfResult"], 1) + assert.Equal(t, explainNode["planExecutions"], uint64(2)) + selectTopNode, ok := explainNode["selectTopNode"].(dataMap) + require.True(t, ok, "Expected selectTopNode") + selectNode, ok := selectTopNode["selectNode"].(dataMap) + require.True(t, ok, "Expected selectNode") + scanNode, ok := selectNode["scanNode"].(dataMap) + require.True(t, ok, "Expected scanNode") + assert.Equal(t, scanNode["iterations"], uint64(a.iterations), + "Expected %d iterations, got %d", a.iterations, scanNode["iterations"]) + assert.Equal(t, scanNode["docFetches"], uint64(a.docFetches), + "Expected %d docFetches, got %d", a.docFetches, scanNode["docFetches"]) + assert.Equal(t, scanNode["filterMatches"], uint64(a.filterMatches), + "Expected %d filterMatches, got %d", a.filterMatches, scanNode["filterMatches"]) +} + +func newExplainAsserter(iterations, docFetched, filterMatcher int) *explainResultAsserter { + return &explainResultAsserter{ + iterations: iterations, + docFetches: docFetched, + filterMatches: filterMatcher, + } +} + +func createUserDocs() []testUtils.CreateDoc { + return []testUtils.CreateDoc{ + { + CollectionID: 0, + Doc: `{ + "name": "John" + }`, + }, + { + CollectionID: 0, + Doc: `{ + "name": "Islam" + }`, + }, + { + CollectionID: 0, + Doc: `{ + "name": "Andy" + }`, + }, + { + CollectionID: 0, + Doc: `{ + "name": "Shahzad" + }`, + }, + { + CollectionID: 0, + Doc: `{ + "name": "Fred" + }`, + }, + { + CollectionID: 0, + Doc: `{ + "name": "Orpheus" + }`, + }, + { + CollectionID: 0, + Doc: `{ + "name": "Addo" + }`, + }, + } +} From f95221007af7f98490f51bf52091436ac8d0318a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 May 2023 10:34:23 +0200 Subject: [PATCH 026/120] Move index tests --- tests/integration/{schema => }/index/simple_test.go | 0 tests/integration/{schema => }/index/utils.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/integration/{schema => }/index/simple_test.go (100%) rename tests/integration/{schema => }/index/utils.go (100%) diff --git a/tests/integration/schema/index/simple_test.go b/tests/integration/index/simple_test.go similarity index 100% rename from tests/integration/schema/index/simple_test.go rename to tests/integration/index/simple_test.go diff --git a/tests/integration/schema/index/utils.go b/tests/integration/index/utils.go similarity index 100% rename from tests/integration/schema/index/utils.go rename to tests/integration/index/utils.go From 71be4167e1a9bb98f5d9e542896a2c2df301f023 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 May 2023 13:42:05 +0200 Subject: [PATCH 027/120] Add IndexDataStoreKey --- core/key.go | 57 +++++++++++++ core/key_test.go | 215 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) diff --git a/core/key.go b/core/key.go index 1713b86e3c..6a66767949 100644 --- a/core/key.go +++ b/core/key.go @@ -68,6 +68,15 @@ type DataStoreKey struct { var _ Key = (*DataStoreKey)(nil) +// IndexDataStoreKey is a type that represents a key of an indexed document in the database. +type IndexDataStoreKey struct { + CollectionID string + IndexID string + FieldValues []string +} + +var _ Key = (*IndexDataStoreKey)(nil) + type PrimaryDataStoreKey struct { CollectionId string DocKey string @@ -342,6 +351,54 @@ func (k DataStoreKey) ToPrimaryDataStoreKey() PrimaryDataStoreKey { } } +func (k *IndexDataStoreKey) Bytes() []byte { + return []byte(k.ToString()) +} + +func (k *IndexDataStoreKey) ToDS() ds.Key { + return ds.NewKey(k.ToString()) +} + +func (k *IndexDataStoreKey) ToString() string { + sb := strings.Builder{} + + if k.CollectionID == "" { + return "" + } + sb.WriteByte('/') + sb.WriteString(k.CollectionID) + + if k.IndexID == "" { + return sb.String() + } + sb.WriteByte('/') + sb.WriteString(k.IndexID) + + for _, v := range k.FieldValues { + if v == "" { + break + } + sb.WriteByte('/') + sb.WriteString(v) + } + + return sb.String() +} + +func (k IndexDataStoreKey) Equal(other IndexDataStoreKey) bool { + if k.CollectionID != other.CollectionID || + k.IndexID != other.IndexID || + len(k.FieldValues) != len(other.FieldValues) { + return false + } + for i := range k.FieldValues { + if k.FieldValues[i] != other.FieldValues[i] { + return false + } + } + return true +} + func (k PrimaryDataStoreKey) ToDataStoreKey() DataStoreKey { return DataStoreKey{ CollectionID: k.CollectionId, diff --git a/core/key_test.go b/core/key_test.go index 3f09efe44d..c5b8836cfe 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -13,6 +13,7 @@ package core import ( "testing" + ds "github.com/ipfs/go-datastore" "github.com/stretchr/testify/assert" ) @@ -155,3 +156,217 @@ func TestNewIndexKeyFromString_IfFullKeyString_ReturnKey(t *testing.T) { assert.Equal(t, key.CollectionName, "col") assert.Equal(t, key.IndexName, "idx") } + +func TestIndexDatastoreKey_ToString(t *testing.T) { + cases := []struct { + Key IndexDataStoreKey + Expected string + }{ + { + Key: IndexDataStoreKey{}, + Expected: "", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + }, + Expected: "/1", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + }, + Expected: "/1/2", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3"}, + }, + Expected: "/1/2/3", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + }, + Expected: "/1/2/3/4", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + FieldValues: []string{"3"}, + }, + Expected: "/1", + }, + { + Key: IndexDataStoreKey{ + IndexID: "2", + FieldValues: []string{"3"}, + }, + Expected: "", + }, + { + Key: IndexDataStoreKey{ + FieldValues: []string{"3"}, + }, + Expected: "", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"", ""}, + }, + Expected: "/1/2", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"", "3"}, + }, + Expected: "/1/2", + }, + { + Key: IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "", "4"}, + }, + Expected: "/1/2/3", + }, + } + for i, c := range cases { + assert.Equal(t, c.Key.ToString(), c.Expected, "case %d", i) + } +} + +func TestIndexDatastoreKey_Bytes(t *testing.T) { + key := IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + } + assert.Equal(t, key.Bytes(), []byte("/1/2/3/4")) +} + +func TestIndexDatastoreKey_ToDS(t *testing.T) { + key := IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + } + assert.Equal(t, key.ToDS(), ds.NewKey("/1/2/3/4")) +} + +func TestIndexDatastoreKey_EqualTrue(t *testing.T) { + cases := [][]IndexDataStoreKey{ + { + { + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + }, + { + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + }, + }, + { + { + CollectionID: "1", + FieldValues: []string{"3", "4"}, + }, + { + CollectionID: "1", + FieldValues: []string{"3", "4"}, + }, + }, + { + { + CollectionID: "1", + }, + { + CollectionID: "1", + }, + }, + } + + for i, c := range cases { + assert.True(t, c[0].Equal(c[1]), "case %d", i) + } +} + +func TestIndexDatastoreKey_EqualFalse(t *testing.T) { + cases := [][]IndexDataStoreKey{ + { + { + CollectionID: "1", + }, + { + CollectionID: "2", + }, + }, + { + { + CollectionID: "1", + IndexID: "2", + }, + { + CollectionID: "1", + IndexID: "3", + }, + }, + { + { + CollectionID: "1", + }, + { + IndexID: "1", + }, + }, + { + { + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"4", "3"}, + }, + { + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + }, + }, + { + { + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3"}, + }, + { + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + }, + }, + { + { + CollectionID: "1", + FieldValues: []string{"3", "", "4"}, + }, + { + CollectionID: "1", + FieldValues: []string{"3", "4"}, + }, + }, + } + + for i, c := range cases { + assert.False(t, c[0].Equal(c[1]), "case %d", i) + } +} From 7356099215d152b36a905541673879776ad10c44 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 May 2023 15:33:27 +0200 Subject: [PATCH 028/120] Add NewIndexDataStoreKey method --- core/key.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ core/key_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/core/key.go b/core/key.go index 6a66767949..136941b086 100644 --- a/core/key.go +++ b/core/key.go @@ -351,6 +351,53 @@ func (k DataStoreKey) ToPrimaryDataStoreKey() PrimaryDataStoreKey { } } +// NewIndexDataStoreKey creates a new IndexDataStoreKey from a string as best as it can, +// splitting the input using '/' as a field deliminator. It assumes +// that the input string is in the following format: +// +// /[CollectionID]/[IndexID]/[FieldID](/[FieldID]...) +// +// Any properties before the above (assuming a '/' deliminator) are ignored +func NewIndexDataStoreKey(key string) (IndexDataStoreKey, error) { + indexKey := IndexDataStoreKey{} + if key == "" { + return indexKey, ErrEmptyKey + } + + if !strings.HasPrefix(key, "/") { + return indexKey, ErrInvalidKey + } + + elements := strings.Split(key[1:], "/") + + // With less than 3 elements, we know it's an invalid key + if len(elements) < 3 { + return indexKey, ErrInvalidKey + } + + _, err := strconv.Atoi(elements[0]) + if err != nil { + return IndexDataStoreKey{}, ErrInvalidKey + } + indexKey.CollectionID = elements[0] + + _, err = strconv.Atoi(elements[1]) + if err != nil { + return IndexDataStoreKey{}, ErrInvalidKey + } + indexKey.IndexID = elements[1] + + for i := 2; i < len(elements); i++ { + _, err = strconv.Atoi(elements[i]) + if err != nil { + return IndexDataStoreKey{}, ErrInvalidKey + } + indexKey.FieldValues = append(indexKey.FieldValues, elements[i]) + } + + return indexKey, nil +} + func (k *IndexDataStoreKey) Bytes() []byte { return []byte(k.ToString()) } diff --git a/core/key_test.go b/core/key_test.go index c5b8836cfe..1ca1b836db 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -370,3 +370,41 @@ func TestIndexDatastoreKey_EqualFalse(t *testing.T) { assert.False(t, c[0].Equal(c[1]), "case %d", i) } } + +func TestNewIndexDataStoreKey_ValidKey(t *testing.T) { + str, err := NewIndexDataStoreKey("/1/2/3") + assert.NoError(t, err) + assert.Equal(t, str, IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3"}, + }) + + str, err = NewIndexDataStoreKey("/1/2/3/4") + assert.NoError(t, err) + assert.Equal(t, str, IndexDataStoreKey{ + CollectionID: "1", + IndexID: "2", + FieldValues: []string{"3", "4"}, + }) +} + +func TestNewIndexDataStoreKey_InvalidKey(t *testing.T) { + keys := []string{ + "", + "/", + "/1", + "/1/2", + " /1/2/3", + "/1/2/3 ", + "1/2/3", + "/a/2/3", + "/1/b/3", + "/1/2/c", + "/1/2/3/d", + } + for i, key := range keys { + _, err := NewIndexDataStoreKey(key) + assert.Error(t, err, "case %d: %s", i, key) + } +} From 6b16651d925c2f4b2962097f8713dccb797d258c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 9 May 2023 15:07:53 +0200 Subject: [PATCH 029/120] Add base implementation of indexing a doc --- db/collection.go | 18 ++++++++++- db/index.go | 19 ++++++++++-- db/index_test.go | 27 ++++++++++------ db/indexed_docs_test.go | 69 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 db/indexed_docs_test.go diff --git a/db/collection.go b/db/collection.go index 5852dfe210..6559067326 100644 --- a/db/collection.go +++ b/db/collection.go @@ -779,7 +779,23 @@ func (c *collection) create(ctx context.Context, txn datastore.Txn, doc *client. return err } - return err + return c.indexNewDoc(ctx, txn, doc) +} + +func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { + colIndexKey := core.NewCollectionIndexKey(c.desc.Name, "user_name") + indexData, err := txn.Systemstore().Get(ctx, colIndexKey.ToDS()) + err = err + var indexDesc client.IndexDescription + err = json.Unmarshal(indexData, &indexDesc) + err = err + colIndex := NewCollectionIndex(c, indexDesc) + docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) + fieldVal, err := doc.Get("name") + err = err + err = colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) + err = err + return nil } // Update an existing document with the new values. diff --git a/db/index.go b/db/index.go index 072f2dd60a..ad2a287bdc 100644 --- a/db/index.go +++ b/db/index.go @@ -11,10 +11,11 @@ import ( "github.com/ipfs/go-datastore/query" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" + "github.com/sourcenetwork/defradb/datastore" ) type CollectionIndex interface { - Save(core.DataStoreKey, client.Value) error + Save(context.Context, datastore.Txn, core.DataStoreKey, any) error Name() string Description() client.IndexDescription } @@ -31,7 +32,21 @@ type collectionSimpleIndex struct { desc client.IndexDescription } -func (c *collectionSimpleIndex) Save(core.DataStoreKey, client.Value) error { +var _ CollectionIndex = (*collectionSimpleIndex)(nil) + +func (c *collectionSimpleIndex) Save( + ctx context.Context, + txn datastore.Txn, + key core.DataStoreKey, + val any, +) error { + data := val.(string) + indexDataStoreKey := core.IndexDataStoreKey{} + indexDataStoreKey.CollectionID = strconv.Itoa(int(c.collection.ID())) + indexDataStoreKey.IndexID = "1" + indexDataStoreKey.FieldValues = []string{data, key.DocKey} + err := txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) + err = err return nil } diff --git a/db/index_test.go b/db/index_test.go index 73b657e9cc..b14c770f08 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -94,9 +94,9 @@ func getProductsCollectionDesc() client.CollectionDescription { func newIndexTestFixture(t *testing.T) *indexTestFixture { ctx := context.Background() db, err := newMemoryDB(ctx) - assert.NoError(t, err) + require.NoError(t, err) txn, err := db.NewTxn(ctx, false) - assert.NoError(t, err) + require.NoError(t, err) f := &indexTestFixture{ ctx: ctx, @@ -114,15 +114,16 @@ func (f *indexTestFixture) createCollectionIndex( return f.createCollectionIndexFor(f.collection.Name(), desc) } -func (f *indexTestFixture) createUserCollectionIndex() client.IndexDescription { +func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescription { desc := client.IndexDescription{ - Name: "some_index_name", + Name: "user_name", Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, } newDesc, err := f.createCollectionIndexFor(f.collection.Name(), desc) - assert.NoError(f.t, err) + require.NoError(f.t, err) + f.commitTxn() return newDesc } @@ -156,6 +157,14 @@ func (f *indexTestFixture) countIndexPrefixes(colName, indexName string) int { return count } +func (f *indexTestFixture) commitTxn() { + err := f.txn.Commit(f.ctx) + require.NoError(f.t, err) + txn, err := f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + f.txn = txn +} + func (f *indexTestFixture) createCollectionIndexFor( collectionName string, desc client.IndexDescription, @@ -459,7 +468,7 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) - f.createUserCollectionIndex() + f.createUserCollectionIndexOnName() f.db.Close(f.ctx) @@ -480,7 +489,7 @@ func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { func TestDropIndex_ShouldDeleteIndex(t *testing.T) { f := newIndexTestFixture(t) - desc := f.createUserCollectionIndex() + desc := f.createUserCollectionIndexOnName() err := f.dropIndex(usersColName, desc.Name) assert.NoError(t, err) @@ -492,7 +501,7 @@ func TestDropIndex_ShouldDeleteIndex(t *testing.T) { func TestDropIndex_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) - desc := f.createUserCollectionIndex() + desc := f.createUserCollectionIndexOnName() f.db.Close(f.ctx) @@ -533,7 +542,7 @@ func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) - f.createUserCollectionIndex() + f.createUserCollectionIndexOnName() f.db.Close(f.ctx) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go new file mode 100644 index 0000000000..51b5472ed7 --- /dev/null +++ b/db/indexed_docs_test.go @@ -0,0 +1,69 @@ +package db + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type userDoc struct { + Name string `json:"name"` + Age int `json:"age"` + Weight float64 `json:"weight"` +} + +func (f *indexTestFixture) createUserDoc(name string, age int) *client.Document { + d := userDoc{Name: name, Age: age, Weight: 154.1} + data, err := json.Marshal(d) + require.NoError(f.t, err) + + doc, err := client.NewDocFromJSON(data) + if err != nil { + f.t.Error(err) + return nil + } + err = f.collection.Create(f.ctx, doc) + require.NoError(f.t, err) + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + return doc +} + +func (f *indexTestFixture) getNonUniqueDocIndex( + doc *client.Document, + fieldName string, +) core.IndexDataStoreKey { + colDesc := f.collection.Description() + field, ok := colDesc.GetField(fieldName) + require.True(f.t, ok) + + fieldVal, err := doc.Get(fieldName) + require.NoError(f.t, err) + fieldStrVal, ok := fieldVal.(string) + require.True(f.t, ok) + + key := core.IndexDataStoreKey{ + CollectionID: strconv.Itoa(int(f.collection.ID())), + IndexID: strconv.Itoa(int(field.ID)), + FieldValues: []string{fieldStrVal, doc.Key().String()}, + } + return key +} + +func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + doc := f.createUserDoc("John", 21) + + key := f.getNonUniqueDocIndex(doc, "name") + + data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} From 9a20691cd5aaa585625f2aafeb79c750196d2400 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 10 May 2023 13:15:26 +0200 Subject: [PATCH 030/120] Create mocks for Txn and related interfaces --- datastore/mocks/DAGStore.go | 417 ++++++++++++++++++++++++++++++ datastore/mocks/DSReaderWriter.go | 400 ++++++++++++++++++++++++++++ datastore/mocks/Results.go | 310 ++++++++++++++++++++++ datastore/mocks/Txn.go | 394 ++++++++++++++++++++++++++++ datastore/mocks/utils.go | 81 ++++++ 5 files changed, 1602 insertions(+) create mode 100644 datastore/mocks/DAGStore.go create mode 100644 datastore/mocks/DSReaderWriter.go create mode 100644 datastore/mocks/Results.go create mode 100644 datastore/mocks/Txn.go create mode 100644 datastore/mocks/utils.go diff --git a/datastore/mocks/DAGStore.go b/datastore/mocks/DAGStore.go new file mode 100644 index 0000000000..c0fff43f27 --- /dev/null +++ b/datastore/mocks/DAGStore.go @@ -0,0 +1,417 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + blocks "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// DAGStore is an autogenerated mock type for the DAGStore type +type DAGStore struct { + mock.Mock +} + +type DAGStore_Expecter struct { + mock *mock.Mock +} + +func (_m *DAGStore) EXPECT() *DAGStore_Expecter { + return &DAGStore_Expecter{mock: &_m.Mock} +} + +// AllKeysChan provides a mock function with given fields: ctx +func (_m *DAGStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + ret := _m.Called(ctx) + + var r0 <-chan cid.Cid + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (<-chan cid.Cid, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) <-chan cid.Cid); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan cid.Cid) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DAGStore_AllKeysChan_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllKeysChan' +type DAGStore_AllKeysChan_Call struct { + *mock.Call +} + +// AllKeysChan is a helper method to define mock.On call +// - ctx context.Context +func (_e *DAGStore_Expecter) AllKeysChan(ctx interface{}) *DAGStore_AllKeysChan_Call { + return &DAGStore_AllKeysChan_Call{Call: _e.mock.On("AllKeysChan", ctx)} +} + +func (_c *DAGStore_AllKeysChan_Call) Run(run func(ctx context.Context)) *DAGStore_AllKeysChan_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *DAGStore_AllKeysChan_Call) Return(_a0 <-chan cid.Cid, _a1 error) *DAGStore_AllKeysChan_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DAGStore_AllKeysChan_Call) RunAndReturn(run func(context.Context) (<-chan cid.Cid, error)) *DAGStore_AllKeysChan_Call { + _c.Call.Return(run) + return _c +} + +// DeleteBlock provides a mock function with given fields: _a0, _a1 +func (_m *DAGStore) DeleteBlock(_a0 context.Context, _a1 cid.Cid) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DAGStore_DeleteBlock_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteBlock' +type DAGStore_DeleteBlock_Call struct { + *mock.Call +} + +// DeleteBlock is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *DAGStore_Expecter) DeleteBlock(_a0 interface{}, _a1 interface{}) *DAGStore_DeleteBlock_Call { + return &DAGStore_DeleteBlock_Call{Call: _e.mock.On("DeleteBlock", _a0, _a1)} +} + +func (_c *DAGStore_DeleteBlock_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *DAGStore_DeleteBlock_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *DAGStore_DeleteBlock_Call) Return(_a0 error) *DAGStore_DeleteBlock_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DAGStore_DeleteBlock_Call) RunAndReturn(run func(context.Context, cid.Cid) error) *DAGStore_DeleteBlock_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function with given fields: _a0, _a1 +func (_m *DAGStore) Get(_a0 context.Context, _a1 cid.Cid) (blocks.Block, error) { + ret := _m.Called(_a0, _a1) + + var r0 blocks.Block + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) (blocks.Block, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) blocks.Block); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(blocks.Block) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, cid.Cid) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DAGStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type DAGStore_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *DAGStore_Expecter) Get(_a0 interface{}, _a1 interface{}) *DAGStore_Get_Call { + return &DAGStore_Get_Call{Call: _e.mock.On("Get", _a0, _a1)} +} + +func (_c *DAGStore_Get_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *DAGStore_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *DAGStore_Get_Call) Return(_a0 blocks.Block, _a1 error) *DAGStore_Get_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DAGStore_Get_Call) RunAndReturn(run func(context.Context, cid.Cid) (blocks.Block, error)) *DAGStore_Get_Call { + _c.Call.Return(run) + return _c +} + +// GetSize provides a mock function with given fields: _a0, _a1 +func (_m *DAGStore) GetSize(_a0 context.Context, _a1 cid.Cid) (int, error) { + ret := _m.Called(_a0, _a1) + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) (int, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) int); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(context.Context, cid.Cid) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DAGStore_GetSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSize' +type DAGStore_GetSize_Call struct { + *mock.Call +} + +// GetSize is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *DAGStore_Expecter) GetSize(_a0 interface{}, _a1 interface{}) *DAGStore_GetSize_Call { + return &DAGStore_GetSize_Call{Call: _e.mock.On("GetSize", _a0, _a1)} +} + +func (_c *DAGStore_GetSize_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *DAGStore_GetSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *DAGStore_GetSize_Call) Return(_a0 int, _a1 error) *DAGStore_GetSize_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DAGStore_GetSize_Call) RunAndReturn(run func(context.Context, cid.Cid) (int, error)) *DAGStore_GetSize_Call { + _c.Call.Return(run) + return _c +} + +// Has provides a mock function with given fields: _a0, _a1 +func (_m *DAGStore) Has(_a0 context.Context, _a1 cid.Cid) (bool, error) { + ret := _m.Called(_a0, _a1) + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) (bool, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) bool); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, cid.Cid) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DAGStore_Has_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Has' +type DAGStore_Has_Call struct { + *mock.Call +} + +// Has is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *DAGStore_Expecter) Has(_a0 interface{}, _a1 interface{}) *DAGStore_Has_Call { + return &DAGStore_Has_Call{Call: _e.mock.On("Has", _a0, _a1)} +} + +func (_c *DAGStore_Has_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *DAGStore_Has_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *DAGStore_Has_Call) Return(_a0 bool, _a1 error) *DAGStore_Has_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DAGStore_Has_Call) RunAndReturn(run func(context.Context, cid.Cid) (bool, error)) *DAGStore_Has_Call { + _c.Call.Return(run) + return _c +} + +// HashOnRead provides a mock function with given fields: enabled +func (_m *DAGStore) HashOnRead(enabled bool) { + _m.Called(enabled) +} + +// DAGStore_HashOnRead_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HashOnRead' +type DAGStore_HashOnRead_Call struct { + *mock.Call +} + +// HashOnRead is a helper method to define mock.On call +// - enabled bool +func (_e *DAGStore_Expecter) HashOnRead(enabled interface{}) *DAGStore_HashOnRead_Call { + return &DAGStore_HashOnRead_Call{Call: _e.mock.On("HashOnRead", enabled)} +} + +func (_c *DAGStore_HashOnRead_Call) Run(run func(enabled bool)) *DAGStore_HashOnRead_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(bool)) + }) + return _c +} + +func (_c *DAGStore_HashOnRead_Call) Return() *DAGStore_HashOnRead_Call { + _c.Call.Return() + return _c +} + +func (_c *DAGStore_HashOnRead_Call) RunAndReturn(run func(bool)) *DAGStore_HashOnRead_Call { + _c.Call.Return(run) + return _c +} + +// Put provides a mock function with given fields: _a0, _a1 +func (_m *DAGStore) Put(_a0 context.Context, _a1 blocks.Block) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, blocks.Block) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DAGStore_Put_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Put' +type DAGStore_Put_Call struct { + *mock.Call +} + +// Put is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 blocks.Block +func (_e *DAGStore_Expecter) Put(_a0 interface{}, _a1 interface{}) *DAGStore_Put_Call { + return &DAGStore_Put_Call{Call: _e.mock.On("Put", _a0, _a1)} +} + +func (_c *DAGStore_Put_Call) Run(run func(_a0 context.Context, _a1 blocks.Block)) *DAGStore_Put_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(blocks.Block)) + }) + return _c +} + +func (_c *DAGStore_Put_Call) Return(_a0 error) *DAGStore_Put_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DAGStore_Put_Call) RunAndReturn(run func(context.Context, blocks.Block) error) *DAGStore_Put_Call { + _c.Call.Return(run) + return _c +} + +// PutMany provides a mock function with given fields: _a0, _a1 +func (_m *DAGStore) PutMany(_a0 context.Context, _a1 []blocks.Block) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, []blocks.Block) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DAGStore_PutMany_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutMany' +type DAGStore_PutMany_Call struct { + *mock.Call +} + +// PutMany is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 []blocks.Block +func (_e *DAGStore_Expecter) PutMany(_a0 interface{}, _a1 interface{}) *DAGStore_PutMany_Call { + return &DAGStore_PutMany_Call{Call: _e.mock.On("PutMany", _a0, _a1)} +} + +func (_c *DAGStore_PutMany_Call) Run(run func(_a0 context.Context, _a1 []blocks.Block)) *DAGStore_PutMany_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]blocks.Block)) + }) + return _c +} + +func (_c *DAGStore_PutMany_Call) Return(_a0 error) *DAGStore_PutMany_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DAGStore_PutMany_Call) RunAndReturn(run func(context.Context, []blocks.Block) error) *DAGStore_PutMany_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewDAGStore interface { + mock.TestingT + Cleanup(func()) +} + +// NewDAGStore creates a new instance of DAGStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDAGStore(t mockConstructorTestingTNewDAGStore) *DAGStore { + mock := &DAGStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/datastore/mocks/DSReaderWriter.go b/datastore/mocks/DSReaderWriter.go new file mode 100644 index 0000000000..459b30037f --- /dev/null +++ b/datastore/mocks/DSReaderWriter.go @@ -0,0 +1,400 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + datastore "github.com/ipfs/go-datastore" + + iterable "github.com/sourcenetwork/defradb/datastore/iterable" + + mock "github.com/stretchr/testify/mock" + + query "github.com/ipfs/go-datastore/query" +) + +// DSReaderWriter is an autogenerated mock type for the DSReaderWriter type +type DSReaderWriter struct { + mock.Mock +} + +type DSReaderWriter_Expecter struct { + mock *mock.Mock +} + +func (_m *DSReaderWriter) EXPECT() *DSReaderWriter_Expecter { + return &DSReaderWriter_Expecter{mock: &_m.Mock} +} + +// Delete provides a mock function with given fields: ctx, key +func (_m *DSReaderWriter) Delete(ctx context.Context, key datastore.Key) error { + ret := _m.Called(ctx, key) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) error); ok { + r0 = rf(ctx, key) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DSReaderWriter_Delete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Delete' +type DSReaderWriter_Delete_Call struct { + *mock.Call +} + +// Delete is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *DSReaderWriter_Expecter) Delete(ctx interface{}, key interface{}) *DSReaderWriter_Delete_Call { + return &DSReaderWriter_Delete_Call{Call: _e.mock.On("Delete", ctx, key)} +} + +func (_c *DSReaderWriter_Delete_Call) Run(run func(ctx context.Context, key datastore.Key)) *DSReaderWriter_Delete_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *DSReaderWriter_Delete_Call) Return(_a0 error) *DSReaderWriter_Delete_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DSReaderWriter_Delete_Call) RunAndReturn(run func(context.Context, datastore.Key) error) *DSReaderWriter_Delete_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function with given fields: ctx, key +func (_m *DSReaderWriter) Get(ctx context.Context, key datastore.Key) ([]byte, error) { + ret := _m.Called(ctx, key) + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) ([]byte, error)); ok { + return rf(ctx, key) + } + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) []byte); ok { + r0 = rf(ctx, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, datastore.Key) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DSReaderWriter_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type DSReaderWriter_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *DSReaderWriter_Expecter) Get(ctx interface{}, key interface{}) *DSReaderWriter_Get_Call { + return &DSReaderWriter_Get_Call{Call: _e.mock.On("Get", ctx, key)} +} + +func (_c *DSReaderWriter_Get_Call) Run(run func(ctx context.Context, key datastore.Key)) *DSReaderWriter_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *DSReaderWriter_Get_Call) Return(value []byte, err error) *DSReaderWriter_Get_Call { + _c.Call.Return(value, err) + return _c +} + +func (_c *DSReaderWriter_Get_Call) RunAndReturn(run func(context.Context, datastore.Key) ([]byte, error)) *DSReaderWriter_Get_Call { + _c.Call.Return(run) + return _c +} + +// GetIterator provides a mock function with given fields: q +func (_m *DSReaderWriter) GetIterator(q query.Query) (iterable.Iterator, error) { + ret := _m.Called(q) + + var r0 iterable.Iterator + var r1 error + if rf, ok := ret.Get(0).(func(query.Query) (iterable.Iterator, error)); ok { + return rf(q) + } + if rf, ok := ret.Get(0).(func(query.Query) iterable.Iterator); ok { + r0 = rf(q) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(iterable.Iterator) + } + } + + if rf, ok := ret.Get(1).(func(query.Query) error); ok { + r1 = rf(q) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DSReaderWriter_GetIterator_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetIterator' +type DSReaderWriter_GetIterator_Call struct { + *mock.Call +} + +// GetIterator is a helper method to define mock.On call +// - q query.Query +func (_e *DSReaderWriter_Expecter) GetIterator(q interface{}) *DSReaderWriter_GetIterator_Call { + return &DSReaderWriter_GetIterator_Call{Call: _e.mock.On("GetIterator", q)} +} + +func (_c *DSReaderWriter_GetIterator_Call) Run(run func(q query.Query)) *DSReaderWriter_GetIterator_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(query.Query)) + }) + return _c +} + +func (_c *DSReaderWriter_GetIterator_Call) Return(_a0 iterable.Iterator, _a1 error) *DSReaderWriter_GetIterator_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DSReaderWriter_GetIterator_Call) RunAndReturn(run func(query.Query) (iterable.Iterator, error)) *DSReaderWriter_GetIterator_Call { + _c.Call.Return(run) + return _c +} + +// GetSize provides a mock function with given fields: ctx, key +func (_m *DSReaderWriter) GetSize(ctx context.Context, key datastore.Key) (int, error) { + ret := _m.Called(ctx, key) + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) (int, error)); ok { + return rf(ctx, key) + } + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) int); ok { + r0 = rf(ctx, key) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(context.Context, datastore.Key) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DSReaderWriter_GetSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSize' +type DSReaderWriter_GetSize_Call struct { + *mock.Call +} + +// GetSize is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *DSReaderWriter_Expecter) GetSize(ctx interface{}, key interface{}) *DSReaderWriter_GetSize_Call { + return &DSReaderWriter_GetSize_Call{Call: _e.mock.On("GetSize", ctx, key)} +} + +func (_c *DSReaderWriter_GetSize_Call) Run(run func(ctx context.Context, key datastore.Key)) *DSReaderWriter_GetSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *DSReaderWriter_GetSize_Call) Return(size int, err error) *DSReaderWriter_GetSize_Call { + _c.Call.Return(size, err) + return _c +} + +func (_c *DSReaderWriter_GetSize_Call) RunAndReturn(run func(context.Context, datastore.Key) (int, error)) *DSReaderWriter_GetSize_Call { + _c.Call.Return(run) + return _c +} + +// Has provides a mock function with given fields: ctx, key +func (_m *DSReaderWriter) Has(ctx context.Context, key datastore.Key) (bool, error) { + ret := _m.Called(ctx, key) + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) (bool, error)); ok { + return rf(ctx, key) + } + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) bool); ok { + r0 = rf(ctx, key) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, datastore.Key) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DSReaderWriter_Has_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Has' +type DSReaderWriter_Has_Call struct { + *mock.Call +} + +// Has is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *DSReaderWriter_Expecter) Has(ctx interface{}, key interface{}) *DSReaderWriter_Has_Call { + return &DSReaderWriter_Has_Call{Call: _e.mock.On("Has", ctx, key)} +} + +func (_c *DSReaderWriter_Has_Call) Run(run func(ctx context.Context, key datastore.Key)) *DSReaderWriter_Has_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *DSReaderWriter_Has_Call) Return(exists bool, err error) *DSReaderWriter_Has_Call { + _c.Call.Return(exists, err) + return _c +} + +func (_c *DSReaderWriter_Has_Call) RunAndReturn(run func(context.Context, datastore.Key) (bool, error)) *DSReaderWriter_Has_Call { + _c.Call.Return(run) + return _c +} + +// Put provides a mock function with given fields: ctx, key, value +func (_m *DSReaderWriter) Put(ctx context.Context, key datastore.Key, value []byte) error { + ret := _m.Called(ctx, key, value) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key, []byte) error); ok { + r0 = rf(ctx, key, value) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DSReaderWriter_Put_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Put' +type DSReaderWriter_Put_Call struct { + *mock.Call +} + +// Put is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +// - value []byte +func (_e *DSReaderWriter_Expecter) Put(ctx interface{}, key interface{}, value interface{}) *DSReaderWriter_Put_Call { + return &DSReaderWriter_Put_Call{Call: _e.mock.On("Put", ctx, key, value)} +} + +func (_c *DSReaderWriter_Put_Call) Run(run func(ctx context.Context, key datastore.Key, value []byte)) *DSReaderWriter_Put_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key), args[2].([]byte)) + }) + return _c +} + +func (_c *DSReaderWriter_Put_Call) Return(_a0 error) *DSReaderWriter_Put_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DSReaderWriter_Put_Call) RunAndReturn(run func(context.Context, datastore.Key, []byte) error) *DSReaderWriter_Put_Call { + _c.Call.Return(run) + return _c +} + +// Query provides a mock function with given fields: ctx, q +func (_m *DSReaderWriter) Query(ctx context.Context, q query.Query) (query.Results, error) { + ret := _m.Called(ctx, q) + + var r0 query.Results + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, query.Query) (query.Results, error)); ok { + return rf(ctx, q) + } + if rf, ok := ret.Get(0).(func(context.Context, query.Query) query.Results); ok { + r0 = rf(ctx, q) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(query.Results) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, query.Query) error); ok { + r1 = rf(ctx, q) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DSReaderWriter_Query_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Query' +type DSReaderWriter_Query_Call struct { + *mock.Call +} + +// Query is a helper method to define mock.On call +// - ctx context.Context +// - q query.Query +func (_e *DSReaderWriter_Expecter) Query(ctx interface{}, q interface{}) *DSReaderWriter_Query_Call { + return &DSReaderWriter_Query_Call{Call: _e.mock.On("Query", ctx, q)} +} + +func (_c *DSReaderWriter_Query_Call) Run(run func(ctx context.Context, q query.Query)) *DSReaderWriter_Query_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(query.Query)) + }) + return _c +} + +func (_c *DSReaderWriter_Query_Call) Return(_a0 query.Results, _a1 error) *DSReaderWriter_Query_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DSReaderWriter_Query_Call) RunAndReturn(run func(context.Context, query.Query) (query.Results, error)) *DSReaderWriter_Query_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewDSReaderWriter interface { + mock.TestingT + Cleanup(func()) +} + +// NewDSReaderWriter creates a new instance of DSReaderWriter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDSReaderWriter(t mockConstructorTestingTNewDSReaderWriter) *DSReaderWriter { + mock := &DSReaderWriter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/datastore/mocks/Results.go b/datastore/mocks/Results.go new file mode 100644 index 0000000000..3314801454 --- /dev/null +++ b/datastore/mocks/Results.go @@ -0,0 +1,310 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + goprocess "github.com/jbenet/goprocess" + mock "github.com/stretchr/testify/mock" + + query "github.com/ipfs/go-datastore/query" +) + +// Results is an autogenerated mock type for the Results type +type Results struct { + mock.Mock +} + +type Results_Expecter struct { + mock *mock.Mock +} + +func (_m *Results) EXPECT() *Results_Expecter { + return &Results_Expecter{mock: &_m.Mock} +} + +// Close provides a mock function with given fields: +func (_m *Results) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Results_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type Results_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +func (_e *Results_Expecter) Close() *Results_Close_Call { + return &Results_Close_Call{Call: _e.mock.On("Close")} +} + +func (_c *Results_Close_Call) Run(run func()) *Results_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Results_Close_Call) Return(_a0 error) *Results_Close_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Results_Close_Call) RunAndReturn(run func() error) *Results_Close_Call { + _c.Call.Return(run) + return _c +} + +// Next provides a mock function with given fields: +func (_m *Results) Next() <-chan query.Result { + ret := _m.Called() + + var r0 <-chan query.Result + if rf, ok := ret.Get(0).(func() <-chan query.Result); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan query.Result) + } + } + + return r0 +} + +// Results_Next_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Next' +type Results_Next_Call struct { + *mock.Call +} + +// Next is a helper method to define mock.On call +func (_e *Results_Expecter) Next() *Results_Next_Call { + return &Results_Next_Call{Call: _e.mock.On("Next")} +} + +func (_c *Results_Next_Call) Run(run func()) *Results_Next_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Results_Next_Call) Return(_a0 <-chan query.Result) *Results_Next_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Results_Next_Call) RunAndReturn(run func() <-chan query.Result) *Results_Next_Call { + _c.Call.Return(run) + return _c +} + +// NextSync provides a mock function with given fields: +func (_m *Results) NextSync() (query.Result, bool) { + ret := _m.Called() + + var r0 query.Result + var r1 bool + if rf, ok := ret.Get(0).(func() (query.Result, bool)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() query.Result); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(query.Result) + } + + if rf, ok := ret.Get(1).(func() bool); ok { + r1 = rf() + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// Results_NextSync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NextSync' +type Results_NextSync_Call struct { + *mock.Call +} + +// NextSync is a helper method to define mock.On call +func (_e *Results_Expecter) NextSync() *Results_NextSync_Call { + return &Results_NextSync_Call{Call: _e.mock.On("NextSync")} +} + +func (_c *Results_NextSync_Call) Run(run func()) *Results_NextSync_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Results_NextSync_Call) Return(_a0 query.Result, _a1 bool) *Results_NextSync_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Results_NextSync_Call) RunAndReturn(run func() (query.Result, bool)) *Results_NextSync_Call { + _c.Call.Return(run) + return _c +} + +// Process provides a mock function with given fields: +func (_m *Results) Process() goprocess.Process { + ret := _m.Called() + + var r0 goprocess.Process + if rf, ok := ret.Get(0).(func() goprocess.Process); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(goprocess.Process) + } + } + + return r0 +} + +// Results_Process_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Process' +type Results_Process_Call struct { + *mock.Call +} + +// Process is a helper method to define mock.On call +func (_e *Results_Expecter) Process() *Results_Process_Call { + return &Results_Process_Call{Call: _e.mock.On("Process")} +} + +func (_c *Results_Process_Call) Run(run func()) *Results_Process_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Results_Process_Call) Return(_a0 goprocess.Process) *Results_Process_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Results_Process_Call) RunAndReturn(run func() goprocess.Process) *Results_Process_Call { + _c.Call.Return(run) + return _c +} + +// Query provides a mock function with given fields: +func (_m *Results) Query() query.Query { + ret := _m.Called() + + var r0 query.Query + if rf, ok := ret.Get(0).(func() query.Query); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(query.Query) + } + + return r0 +} + +// Results_Query_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Query' +type Results_Query_Call struct { + *mock.Call +} + +// Query is a helper method to define mock.On call +func (_e *Results_Expecter) Query() *Results_Query_Call { + return &Results_Query_Call{Call: _e.mock.On("Query")} +} + +func (_c *Results_Query_Call) Run(run func()) *Results_Query_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Results_Query_Call) Return(_a0 query.Query) *Results_Query_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Results_Query_Call) RunAndReturn(run func() query.Query) *Results_Query_Call { + _c.Call.Return(run) + return _c +} + +// Rest provides a mock function with given fields: +func (_m *Results) Rest() ([]query.Entry, error) { + ret := _m.Called() + + var r0 []query.Entry + var r1 error + if rf, ok := ret.Get(0).(func() ([]query.Entry, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []query.Entry); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]query.Entry) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Results_Rest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Rest' +type Results_Rest_Call struct { + *mock.Call +} + +// Rest is a helper method to define mock.On call +func (_e *Results_Expecter) Rest() *Results_Rest_Call { + return &Results_Rest_Call{Call: _e.mock.On("Rest")} +} + +func (_c *Results_Rest_Call) Run(run func()) *Results_Rest_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Results_Rest_Call) Return(_a0 []query.Entry, _a1 error) *Results_Rest_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Results_Rest_Call) RunAndReturn(run func() ([]query.Entry, error)) *Results_Rest_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewResults interface { + mock.TestingT + Cleanup(func()) +} + +// NewResults creates a new instance of Results. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewResults(t mockConstructorTestingTNewResults) *Results { + mock := &Results{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/datastore/mocks/Txn.go b/datastore/mocks/Txn.go new file mode 100644 index 0000000000..c190be999e --- /dev/null +++ b/datastore/mocks/Txn.go @@ -0,0 +1,394 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + datastore "github.com/sourcenetwork/defradb/datastore" + mock "github.com/stretchr/testify/mock" +) + +// Txn is an autogenerated mock type for the Txn type +type Txn struct { + mock.Mock +} + +type Txn_Expecter struct { + mock *mock.Mock +} + +func (_m *Txn) EXPECT() *Txn_Expecter { + return &Txn_Expecter{mock: &_m.Mock} +} + +// Commit provides a mock function with given fields: ctx +func (_m *Txn) Commit(ctx context.Context) error { + ret := _m.Called(ctx) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(ctx) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Txn_Commit_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Commit' +type Txn_Commit_Call struct { + *mock.Call +} + +// Commit is a helper method to define mock.On call +// - ctx context.Context +func (_e *Txn_Expecter) Commit(ctx interface{}) *Txn_Commit_Call { + return &Txn_Commit_Call{Call: _e.mock.On("Commit", ctx)} +} + +func (_c *Txn_Commit_Call) Run(run func(ctx context.Context)) *Txn_Commit_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Txn_Commit_Call) Return(_a0 error) *Txn_Commit_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Txn_Commit_Call) RunAndReturn(run func(context.Context) error) *Txn_Commit_Call { + _c.Call.Return(run) + return _c +} + +// DAGstore provides a mock function with given fields: +func (_m *Txn) DAGstore() datastore.DAGStore { + ret := _m.Called() + + var r0 datastore.DAGStore + if rf, ok := ret.Get(0).(func() datastore.DAGStore); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.DAGStore) + } + } + + return r0 +} + +// Txn_DAGstore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DAGstore' +type Txn_DAGstore_Call struct { + *mock.Call +} + +// DAGstore is a helper method to define mock.On call +func (_e *Txn_Expecter) DAGstore() *Txn_DAGstore_Call { + return &Txn_DAGstore_Call{Call: _e.mock.On("DAGstore")} +} + +func (_c *Txn_DAGstore_Call) Run(run func()) *Txn_DAGstore_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Txn_DAGstore_Call) Return(_a0 datastore.DAGStore) *Txn_DAGstore_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Txn_DAGstore_Call) RunAndReturn(run func() datastore.DAGStore) *Txn_DAGstore_Call { + _c.Call.Return(run) + return _c +} + +// Datastore provides a mock function with given fields: +func (_m *Txn) Datastore() datastore.DSReaderWriter { + ret := _m.Called() + + var r0 datastore.DSReaderWriter + if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.DSReaderWriter) + } + } + + return r0 +} + +// Txn_Datastore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Datastore' +type Txn_Datastore_Call struct { + *mock.Call +} + +// Datastore is a helper method to define mock.On call +func (_e *Txn_Expecter) Datastore() *Txn_Datastore_Call { + return &Txn_Datastore_Call{Call: _e.mock.On("Datastore")} +} + +func (_c *Txn_Datastore_Call) Run(run func()) *Txn_Datastore_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Txn_Datastore_Call) Return(_a0 datastore.DSReaderWriter) *Txn_Datastore_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Txn_Datastore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *Txn_Datastore_Call { + _c.Call.Return(run) + return _c +} + +// Discard provides a mock function with given fields: ctx +func (_m *Txn) Discard(ctx context.Context) { + _m.Called(ctx) +} + +// Txn_Discard_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Discard' +type Txn_Discard_Call struct { + *mock.Call +} + +// Discard is a helper method to define mock.On call +// - ctx context.Context +func (_e *Txn_Expecter) Discard(ctx interface{}) *Txn_Discard_Call { + return &Txn_Discard_Call{Call: _e.mock.On("Discard", ctx)} +} + +func (_c *Txn_Discard_Call) Run(run func(ctx context.Context)) *Txn_Discard_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Txn_Discard_Call) Return() *Txn_Discard_Call { + _c.Call.Return() + return _c +} + +func (_c *Txn_Discard_Call) RunAndReturn(run func(context.Context)) *Txn_Discard_Call { + _c.Call.Return(run) + return _c +} + +// Headstore provides a mock function with given fields: +func (_m *Txn) Headstore() datastore.DSReaderWriter { + ret := _m.Called() + + var r0 datastore.DSReaderWriter + if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.DSReaderWriter) + } + } + + return r0 +} + +// Txn_Headstore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Headstore' +type Txn_Headstore_Call struct { + *mock.Call +} + +// Headstore is a helper method to define mock.On call +func (_e *Txn_Expecter) Headstore() *Txn_Headstore_Call { + return &Txn_Headstore_Call{Call: _e.mock.On("Headstore")} +} + +func (_c *Txn_Headstore_Call) Run(run func()) *Txn_Headstore_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Txn_Headstore_Call) Return(_a0 datastore.DSReaderWriter) *Txn_Headstore_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Txn_Headstore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *Txn_Headstore_Call { + _c.Call.Return(run) + return _c +} + +// OnError provides a mock function with given fields: fn +func (_m *Txn) OnError(fn func()) { + _m.Called(fn) +} + +// Txn_OnError_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OnError' +type Txn_OnError_Call struct { + *mock.Call +} + +// OnError is a helper method to define mock.On call +// - fn func() +func (_e *Txn_Expecter) OnError(fn interface{}) *Txn_OnError_Call { + return &Txn_OnError_Call{Call: _e.mock.On("OnError", fn)} +} + +func (_c *Txn_OnError_Call) Run(run func(fn func())) *Txn_OnError_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(func())) + }) + return _c +} + +func (_c *Txn_OnError_Call) Return() *Txn_OnError_Call { + _c.Call.Return() + return _c +} + +func (_c *Txn_OnError_Call) RunAndReturn(run func(func())) *Txn_OnError_Call { + _c.Call.Return(run) + return _c +} + +// OnSuccess provides a mock function with given fields: fn +func (_m *Txn) OnSuccess(fn func()) { + _m.Called(fn) +} + +// Txn_OnSuccess_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OnSuccess' +type Txn_OnSuccess_Call struct { + *mock.Call +} + +// OnSuccess is a helper method to define mock.On call +// - fn func() +func (_e *Txn_Expecter) OnSuccess(fn interface{}) *Txn_OnSuccess_Call { + return &Txn_OnSuccess_Call{Call: _e.mock.On("OnSuccess", fn)} +} + +func (_c *Txn_OnSuccess_Call) Run(run func(fn func())) *Txn_OnSuccess_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(func())) + }) + return _c +} + +func (_c *Txn_OnSuccess_Call) Return() *Txn_OnSuccess_Call { + _c.Call.Return() + return _c +} + +func (_c *Txn_OnSuccess_Call) RunAndReturn(run func(func())) *Txn_OnSuccess_Call { + _c.Call.Return(run) + return _c +} + +// Rootstore provides a mock function with given fields: +func (_m *Txn) Rootstore() datastore.DSReaderWriter { + ret := _m.Called() + + var r0 datastore.DSReaderWriter + if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.DSReaderWriter) + } + } + + return r0 +} + +// Txn_Rootstore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Rootstore' +type Txn_Rootstore_Call struct { + *mock.Call +} + +// Rootstore is a helper method to define mock.On call +func (_e *Txn_Expecter) Rootstore() *Txn_Rootstore_Call { + return &Txn_Rootstore_Call{Call: _e.mock.On("Rootstore")} +} + +func (_c *Txn_Rootstore_Call) Run(run func()) *Txn_Rootstore_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Txn_Rootstore_Call) Return(_a0 datastore.DSReaderWriter) *Txn_Rootstore_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Txn_Rootstore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *Txn_Rootstore_Call { + _c.Call.Return(run) + return _c +} + +// Systemstore provides a mock function with given fields: +func (_m *Txn) Systemstore() datastore.DSReaderWriter { + ret := _m.Called() + + var r0 datastore.DSReaderWriter + if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.DSReaderWriter) + } + } + + return r0 +} + +// Txn_Systemstore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Systemstore' +type Txn_Systemstore_Call struct { + *mock.Call +} + +// Systemstore is a helper method to define mock.On call +func (_e *Txn_Expecter) Systemstore() *Txn_Systemstore_Call { + return &Txn_Systemstore_Call{Call: _e.mock.On("Systemstore")} +} + +func (_c *Txn_Systemstore_Call) Run(run func()) *Txn_Systemstore_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Txn_Systemstore_Call) Return(_a0 datastore.DSReaderWriter) *Txn_Systemstore_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Txn_Systemstore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *Txn_Systemstore_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewTxn interface { + mock.TestingT + Cleanup(func()) +} + +// NewTxn creates a new instance of Txn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTxn(t mockConstructorTestingTNewTxn) *Txn { + mock := &Txn{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go new file mode 100644 index 0000000000..607b142820 --- /dev/null +++ b/datastore/mocks/utils.go @@ -0,0 +1,81 @@ +package mocks + +import ( + "testing" + + ds "github.com/ipfs/go-datastore" + query "github.com/ipfs/go-datastore/query" + mock "github.com/stretchr/testify/mock" +) + +type MultiStoreTxn struct { + *Txn + MockRootstore *DSReaderWriter + MockDatastore *DSReaderWriter + MockHeadstore *DSReaderWriter + MockDAGstore *DAGStore + MockSystemstore *DSReaderWriter +} + +func prepareDataStore(t *testing.T) *DSReaderWriter { + dataStore := NewDSReaderWriter(t) + dataStore.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte{}, ds.ErrNotFound).Maybe() + dataStore.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + dataStore.EXPECT().Has(mock.Anything, mock.Anything).Return(true, nil).Maybe() + return dataStore +} + +func prepareRootStore(t *testing.T) *DSReaderWriter { + return NewDSReaderWriter(t) +} + +func prepareHeadStore(t *testing.T) *DSReaderWriter { + headStore := NewDSReaderWriter(t) + + queryResults := NewResults(t) + resultChan := make(chan query.Result) + close(resultChan) + queryResults.EXPECT().Next().Return(resultChan).Maybe() + queryResults.EXPECT().Close().Return(nil).Maybe() + headStore.EXPECT().Query(mock.Anything, mock.Anything).Return(queryResults, nil).Maybe() + + headStore.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte{}, ds.ErrNotFound).Maybe() + headStore.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + headStore.EXPECT().Has(mock.Anything, mock.Anything).Return(false, nil).Maybe() + return headStore +} + +func prepareSystemStore(t *testing.T) *DSReaderWriter { + systemStore := NewDSReaderWriter(t) + systemStore.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte{}, nil).Maybe() + return systemStore +} + +func prepareDAGStore(t *testing.T) *DAGStore { + dagStore := NewDAGStore(t) + dagStore.EXPECT().Put(mock.Anything, mock.Anything).Return(nil).Maybe() + dagStore.EXPECT().Has(mock.Anything, mock.Anything).Return(false, nil).Maybe() + return dagStore +} + +func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { + txn := NewTxn(t) + txn.EXPECT().OnSuccess(mock.Anything) + + result := &MultiStoreTxn{ + Txn: txn, + MockRootstore: prepareRootStore(t), + MockDatastore: prepareDataStore(t), + MockHeadstore: prepareHeadStore(t), + MockDAGstore: prepareDAGStore(t), + MockSystemstore: prepareSystemStore(t), + } + + txn.EXPECT().Rootstore().Return(result.MockRootstore).Maybe() + txn.EXPECT().Datastore().Return(result.MockDatastore).Maybe() + txn.EXPECT().Headstore().Return(result.MockHeadstore).Maybe() + txn.EXPECT().DAGstore().Return(result.MockDAGstore).Maybe() + txn.EXPECT().Systemstore().Return(result.MockSystemstore).Maybe() + + return result +} From cbbdbfb6aa485787ffe4dbc3a0a9e2b8d9a5a82f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 10 May 2023 13:16:41 +0200 Subject: [PATCH 031/120] Add test for index that uses mocks --- db/collection.go | 4 +--- db/index.go | 4 +--- db/index_test.go | 29 ++++++++++++++++----------- db/indexed_docs_test.go | 43 ++++++++++++++++++++++++++++++----------- 4 files changed, 52 insertions(+), 28 deletions(-) diff --git a/db/collection.go b/db/collection.go index 6559067326..eda6e9b628 100644 --- a/db/collection.go +++ b/db/collection.go @@ -793,9 +793,7 @@ func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *cl docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) fieldVal, err := doc.Get("name") err = err - err = colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) - err = err - return nil + return colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) } // Update an existing document with the new values. diff --git a/db/index.go b/db/index.go index ad2a287bdc..827e08c08b 100644 --- a/db/index.go +++ b/db/index.go @@ -45,9 +45,7 @@ func (c *collectionSimpleIndex) Save( indexDataStoreKey.CollectionID = strconv.Itoa(int(c.collection.ID())) indexDataStoreKey.IndexID = "1" indexDataStoreKey.FieldValues = []string{data, key.DocKey} - err := txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) - err = err - return nil + return txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) } func (c *collectionSimpleIndex) Name() string { diff --git a/db/index_test.go b/db/index_test.go index b14c770f08..959ffe501f 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -23,6 +23,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/datastore/mocks" ) const ( @@ -31,11 +32,11 @@ const ( ) type indexTestFixture struct { - ctx context.Context - db *implicitTxnDB - txn datastore.Txn - collection client.Collection - t *testing.T + ctx context.Context + db *implicitTxnDB + txn datastore.Txn + users client.Collection + t *testing.T } func getUsersCollectionDesc() client.CollectionDescription { @@ -104,14 +105,14 @@ func newIndexTestFixture(t *testing.T) *indexTestFixture { txn: txn, t: t, } - f.collection = f.createCollection(getUsersCollectionDesc()) + f.users = f.createCollection(getUsersCollectionDesc()) return f } func (f *indexTestFixture) createCollectionIndex( desc client.IndexDescription, ) (client.IndexDescription, error) { - return f.createCollectionIndexFor(f.collection.Name(), desc) + return f.createCollectionIndexFor(f.users.Name(), desc) } func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescription { @@ -121,18 +122,24 @@ func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescrip {Name: "name", Direction: client.Ascending}, }, } - newDesc, err := f.createCollectionIndexFor(f.collection.Name(), desc) + newDesc, err := f.createCollectionIndexFor(f.users.Name(), desc) require.NoError(f.t, err) f.commitTxn() return newDesc } +func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { + mockTxn := mocks.NewTxnWithMultistore(f.t) + f.txn = mockTxn + return mockTxn +} + func (f *indexTestFixture) dropIndex(colName, indexName string) error { return f.db.dropCollectionIndex(f.ctx, f.txn, colName, indexName) } func (f *indexTestFixture) dropAllIndexes(colName string) error { - col := (f.collection.WithTxn(f.txn)).(*collection) + col := (f.users.WithTxn(f.txn)).(*collection) return col.dropAllIndexes(f.ctx) } @@ -256,7 +263,7 @@ func TestCreateIndex_IfNameIsNotSpecified_GenerateWithLowerCase(t *testing.T) { {Name: "Name", Direction: client.Ascending}, }, } - f.collection.Description().Schema.Fields[1].Name = "Name" + f.users.Description().Schema.Fields[1].Name = "Name" newDesc, err := f.createCollectionIndex(desc) assert.NoError(t, err) assert.Equal(t, newDesc.Name, "users_name_ASC") @@ -328,7 +335,7 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { _, err := f.createCollectionIndex(desc) assert.NoError(t, err) - key := core.NewCollectionIndexKey(f.collection.Name(), name) + key := core.NewCollectionIndexKey(f.users.Name(), name) data, err := f.txn.Systemstore().Get(f.ctx, key.ToDS()) assert.NoError(t, err) var deserialized client.IndexDescription diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 51b5472ed7..5d78f7df42 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -2,12 +2,14 @@ package db import ( "encoding/json" + "errors" "strconv" "testing" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -17,19 +19,19 @@ type userDoc struct { Weight float64 `json:"weight"` } -func (f *indexTestFixture) createUserDoc(name string, age int) *client.Document { +func (f *indexTestFixture) saveToUsers(doc *client.Document) { + err := f.users.Create(f.ctx, doc) + require.NoError(f.t, err) + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) +} + +func (f *indexTestFixture) newUserDoc(name string, age int) *client.Document { d := userDoc{Name: name, Age: age, Weight: 154.1} data, err := json.Marshal(d) require.NoError(f.t, err) doc, err := client.NewDocFromJSON(data) - if err != nil { - f.t.Error(err) - return nil - } - err = f.collection.Create(f.ctx, doc) - require.NoError(f.t, err) - f.txn, err = f.db.NewTxn(f.ctx, false) require.NoError(f.t, err) return doc } @@ -38,7 +40,7 @@ func (f *indexTestFixture) getNonUniqueDocIndex( doc *client.Document, fieldName string, ) core.IndexDataStoreKey { - colDesc := f.collection.Description() + colDesc := f.users.Description() field, ok := colDesc.GetField(fieldName) require.True(f.t, ok) @@ -48,7 +50,7 @@ func (f *indexTestFixture) getNonUniqueDocIndex( require.True(f.t, ok) key := core.IndexDataStoreKey{ - CollectionID: strconv.Itoa(int(f.collection.ID())), + CollectionID: strconv.Itoa(int(f.users.ID())), IndexID: strconv.Itoa(int(field.ID)), FieldValues: []string{fieldStrVal, doc.Key().String()}, } @@ -59,7 +61,8 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() - doc := f.createUserDoc("John", 21) + doc := f.newUserDoc("John", 21) + f.saveToUsers(doc) key := f.getNonUniqueDocIndex(doc, "name") @@ -67,3 +70,21 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { require.NoError(t, err) assert.Len(t, data, 0) } + +func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + doc := f.newUserDoc("John", 21) + key := f.getNonUniqueDocIndex(doc, "name") + + mockTxn := f.mockTxn() + mockTxn.EXPECT().Rootstore().Unset() + expect := mockTxn.MockDatastore.EXPECT() + expect.Put(mock.Anything, mock.Anything, mock.Anything).Unset() + expect.Put(mock.Anything, key.ToDS(), mock.Anything).Return(errors.New("error")) + expect.Put(mock.Anything, mock.Anything, mock.Anything).Return(nil) + + err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) + require.Error(f.t, err) +} From fa4355acd213867da930afc8ba6752278c1ab99d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 10 May 2023 13:52:41 +0200 Subject: [PATCH 032/120] Skip indexing if doc has no field --- db/collection.go | 5 +++- db/indexed_docs_test.go | 63 ++++++++++++++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/db/collection.go b/db/collection.go index eda6e9b628..1fe9c12696 100644 --- a/db/collection.go +++ b/db/collection.go @@ -792,7 +792,10 @@ func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *cl colIndex := NewCollectionIndex(c, indexDesc) docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) fieldVal, err := doc.Get("name") - err = err + // if new doc doesn't have the index field, we don't need to index it + if err != nil { + return nil + } return colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 5d78f7df42..9b645597e7 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -6,6 +6,7 @@ import ( "strconv" "testing" + "github.com/ipfs/go-datastore/query" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/stretchr/testify/assert" @@ -36,27 +37,46 @@ func (f *indexTestFixture) newUserDoc(name string, age int) *client.Document { return doc } -func (f *indexTestFixture) getNonUniqueDocIndex( - doc *client.Document, - fieldName string, -) core.IndexDataStoreKey { +func (f *indexTestFixture) getNonUniqueIndexKey(fieldName string) core.IndexDataStoreKey { colDesc := f.users.Description() field, ok := colDesc.GetField(fieldName) require.True(f.t, ok) + key := core.IndexDataStoreKey{ + CollectionID: strconv.Itoa(int(f.users.ID())), + IndexID: strconv.Itoa(int(field.ID)), + } + return key +} + +func (f *indexTestFixture) getNonUniqueDocIndexKey( + doc *client.Document, + fieldName string, +) core.IndexDataStoreKey { + key := f.getNonUniqueIndexKey(fieldName) + fieldVal, err := doc.Get(fieldName) require.NoError(f.t, err) fieldStrVal, ok := fieldVal.(string) require.True(f.t, ok) - key := core.IndexDataStoreKey{ - CollectionID: strconv.Itoa(int(f.users.ID())), - IndexID: strconv.Itoa(int(field.ID)), - FieldValues: []string{fieldStrVal, doc.Key().String()}, - } + key.FieldValues = []string{fieldStrVal, doc.Key().String()} + return key } +func (f *indexTestFixture) getPrefixFromDataStore(prefix string) [][]byte { + q := query.Query{Prefix: prefix} + res, err := f.txn.Datastore().Query(f.ctx, q) + require.NoError(f.t, err) + + var keys [][]byte + for r := range res.Next() { + keys = append(keys, r.Entry.Value) + } + return keys +} + func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -64,7 +84,7 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveToUsers(doc) - key := f.getNonUniqueDocIndex(doc, "name") + key := f.getNonUniqueDocIndexKey(doc, "name") data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -76,7 +96,7 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21) - key := f.getNonUniqueDocIndex(doc, "name") + key := f.getNonUniqueDocIndexKey(doc, "name") mockTxn := f.mockTxn() mockTxn.EXPECT().Rootstore().Unset() @@ -88,3 +108,24 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) require.Error(f.t, err) } + +func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + data, err := json.Marshal(struct { + Age int `json:"age"` + Weight float64 `json:"weight"` + }{Age: 21, Weight: 154.1}) + require.NoError(f.t, err) + + doc, err := client.NewDocFromJSON(data) + require.NoError(f.t, err) + + err = f.users.Create(f.ctx, doc) + require.NoError(f.t, err) + + key := f.getNonUniqueIndexKey("name") + prefixes := f.getPrefixFromDataStore(key.ToString()) + assert.Len(t, prefixes, 0) +} From 2117bf9815ac4ed748cc63496e604ab22920f6ab Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 10 May 2023 15:41:38 +0200 Subject: [PATCH 033/120] Check if system storage to read index description --- db/collection.go | 12 +++++--- db/errors.go | 13 +++++++++ db/index.go | 6 +++- db/index_test.go | 20 ++++++------- db/indexed_docs_test.go | 64 +++++++++++++++++++++++++++++++++++++---- 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/db/collection.go b/db/collection.go index 1fe9c12696..b15bfe9d8b 100644 --- a/db/collection.go +++ b/db/collection.go @@ -785,17 +785,21 @@ func (c *collection) create(ctx context.Context, txn datastore.Txn, doc *client. func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { colIndexKey := core.NewCollectionIndexKey(c.desc.Name, "user_name") indexData, err := txn.Systemstore().Get(ctx, colIndexKey.ToDS()) - err = err + if err != nil { + return NewErrFailedToReadStoredIndexDesc(err) + } var indexDesc client.IndexDescription err = json.Unmarshal(indexData, &indexDesc) - err = err - colIndex := NewCollectionIndex(c, indexDesc) - docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) + if err != nil { + return NewErrInvalidStoredIndex(err) + } fieldVal, err := doc.Get("name") // if new doc doesn't have the index field, we don't need to index it if err != nil { return nil } + colIndex := NewCollectionIndex(c, indexDesc) + docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) return colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) } diff --git a/db/errors.go b/db/errors.go index ee28aeeb88..0e6aa019ce 100644 --- a/db/errors.go +++ b/db/errors.go @@ -46,6 +46,8 @@ const ( errInvalidStoredIndexKey string = "invalid stored index key" errNonExistingFieldForIndex string = "creating an index on a non-existing property" errCollectionDoesntExisting string = "collection with given name doesn't exist" + errFailedToStoreIndexedField string = "failed to store indexed field" + errFailedToReadStoredIndexDesc string = "failed to read stored index description" ) var ( @@ -135,6 +137,17 @@ func NewErrCollectionDoesntExist(colName string) error { return errors.New(errCollectionDoesntExisting, errors.NewKV("Collection", colName)) } +// NewErrFailedToStoreIndexedField returns a new error indicating that the indexed field could not be stored. +func NewErrFailedToStoreIndexedField(fieldName string, inner error) error { + return errors.Wrap(errFailedToStoreIndexedField, inner, errors.NewKV("Field", fieldName)) +} + +// NewErrFailedToReadStoredIndexDesc returns a new error indicating that the stored index description +// could not be read. +func NewErrFailedToReadStoredIndexDesc(inner error) error { + return errors.Wrap(errFailedToReadStoredIndexDesc, inner) +} + // NewErrFailedToGetCollection returns a new error indicating that the collection could not be obtained. func NewErrFailedToGetCollection(name string, inner error) error { return errors.Wrap(errFailedToGetCollection, inner, errors.NewKV("Name", name)) diff --git a/db/index.go b/db/index.go index 827e08c08b..432ad4a7c4 100644 --- a/db/index.go +++ b/db/index.go @@ -45,7 +45,11 @@ func (c *collectionSimpleIndex) Save( indexDataStoreKey.CollectionID = strconv.Itoa(int(c.collection.ID())) indexDataStoreKey.IndexID = "1" indexDataStoreKey.FieldValues = []string{data, key.DocKey} - return txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) + err := txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) + if err != nil { + return NewErrFailedToStoreIndexedField("name", err) + } + return nil } func (c *collectionSimpleIndex) Name() string { diff --git a/db/index_test.go b/db/index_test.go index 959ffe501f..21c8782fcd 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -23,12 +23,13 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/datastore/mocks" ) const ( usersColName = "Users" productsColName = "Products" + + testUsersColIndexName = "user_name" ) type indexTestFixture struct { @@ -115,25 +116,22 @@ func (f *indexTestFixture) createCollectionIndex( return f.createCollectionIndexFor(f.users.Name(), desc) } -func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescription { - desc := client.IndexDescription{ - Name: "user_name", +func getUsersIndexDescOnName() client.IndexDescription { + return client.IndexDescription{ + Name: testUsersColIndexName, Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, } - newDesc, err := f.createCollectionIndexFor(f.users.Name(), desc) +} + +func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescription { + newDesc, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnName()) require.NoError(f.t, err) f.commitTxn() return newDesc } -func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { - mockTxn := mocks.NewTxnWithMultistore(f.t) - f.txn = mockTxn - return mockTxn -} - func (f *indexTestFixture) dropIndex(colName, indexName string) error { return f.db.dropCollectionIndex(f.ctx, f.txn, colName, indexName) } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 9b645597e7..36a1841b69 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/go-datastore/query" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" + "github.com/sourcenetwork/defradb/datastore/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -77,6 +78,23 @@ func (f *indexTestFixture) getPrefixFromDataStore(prefix string) [][]byte { return keys } +func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { + mockedTxn := mocks.NewTxnWithMultistore(f.t) + + indexDesc := getUsersIndexDescOnName() + indexDescData, err := json.Marshal(indexDesc) + require.NoError(f.t, err) + + systemStoreOn := mockedTxn.MockSystemstore.EXPECT() + systemStoreOn.Get(mock.Anything, mock.Anything).Unset() + colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, indexDesc.Name) + systemStoreOn.Get(mock.Anything, colIndexKey.ToDS()).Return(indexDescData, nil) + systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + + f.txn = mockedTxn + return mockedTxn +} + func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -99,14 +117,14 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { key := f.getNonUniqueDocIndexKey(doc, "name") mockTxn := f.mockTxn() - mockTxn.EXPECT().Rootstore().Unset() - expect := mockTxn.MockDatastore.EXPECT() - expect.Put(mock.Anything, mock.Anything, mock.Anything).Unset() - expect.Put(mock.Anything, key.ToDS(), mock.Anything).Return(errors.New("error")) - expect.Put(mock.Anything, mock.Anything, mock.Anything).Return(nil) + + dataStoreOn := mockTxn.MockDatastore.EXPECT() + dataStoreOn.Put(mock.Anything, mock.Anything, mock.Anything).Unset() + dataStoreOn.Put(mock.Anything, key.ToDS(), mock.Anything).Return(errors.New("error")) + dataStoreOn.Put(mock.Anything, mock.Anything, mock.Anything).Return(nil) err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) - require.Error(f.t, err) + require.ErrorIs(f.t, err, NewErrFailedToStoreIndexedField("name", nil)) } func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { @@ -129,3 +147,37 @@ func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { prefixes := f.getPrefixFromDataStore(key.ToString()) assert.Len(t, prefixes, 0) } + +func TestNonUnique_IfSystemStorageHasInvalidIndexDescription_Error(t *testing.T) { + f := newIndexTestFixture(t) + indexDesc := f.createUserCollectionIndexOnName() + + doc := f.newUserDoc("John", 21) + + mockTxn := f.mockTxn() + systemStoreOn := mockTxn.MockSystemstore.EXPECT() + systemStoreOn.Get(mock.Anything, mock.Anything).Unset() + colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, indexDesc.Name) + systemStoreOn.Get(mock.Anything, colIndexKey.ToDS()).Return([]byte("invalid"), nil) + systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + + err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) + require.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) +} + +func TestNonUnique_IfSystemStorageFailsToReadIndexDesc_Error(t *testing.T) { + f := newIndexTestFixture(t) + indexDesc := f.createUserCollectionIndexOnName() + + doc := f.newUserDoc("John", 21) + + mockTxn := f.mockTxn() + systemStoreOn := mockTxn.MockSystemstore.EXPECT() + systemStoreOn.Get(mock.Anything, mock.Anything).Unset() + colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, indexDesc.Name) + systemStoreOn.Get(mock.Anything, colIndexKey.ToDS()).Return([]byte{}, errors.New("error")) + systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + + err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) + require.ErrorIs(t, err, NewErrFailedToReadStoredIndexDesc(nil)) +} From 91de27585e637e7a70ce4c73b251a3e6731446a1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 11 May 2023 13:27:44 +0200 Subject: [PATCH 034/120] Index int fields --- datastore/mocks/utils.go | 20 +++++++++++----- db/collection.go | 39 ++++++++++++++++-------------- db/index.go | 42 +++++++++++++++++++++++---------- db/index_test.go | 17 ++++++++++++++ db/indexed_docs_test.go | 51 +++++++++++++++++++++++++++------------- 5 files changed, 118 insertions(+), 51 deletions(-) diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go index 607b142820..a4875fb22e 100644 --- a/datastore/mocks/utils.go +++ b/datastore/mocks/utils.go @@ -32,12 +32,8 @@ func prepareRootStore(t *testing.T) *DSReaderWriter { func prepareHeadStore(t *testing.T) *DSReaderWriter { headStore := NewDSReaderWriter(t) - queryResults := NewResults(t) - resultChan := make(chan query.Result) - close(resultChan) - queryResults.EXPECT().Next().Return(resultChan).Maybe() - queryResults.EXPECT().Close().Return(nil).Maybe() - headStore.EXPECT().Query(mock.Anything, mock.Anything).Return(queryResults, nil).Maybe() + headStore.EXPECT().Query(mock.Anything, mock.Anything). + Return(NewQueryResultsWithValues(t), nil).Maybe() headStore.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte{}, ds.ErrNotFound).Maybe() headStore.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() @@ -79,3 +75,15 @@ func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { return result } + +func NewQueryResultsWithValues(t *testing.T, entries ...[]byte) *Results { + queryResults := NewResults(t) + resultChan := make(chan query.Result, len(entries)) + for _, entry := range entries { + resultChan <- query.Result{Entry: query.Entry{Value: entry}} + } + close(resultChan) + queryResults.EXPECT().Next().Return(resultChan).Maybe() + queryResults.EXPECT().Close().Return(nil).Maybe() + return queryResults +} diff --git a/db/collection.go b/db/collection.go index b15bfe9d8b..d317488ed9 100644 --- a/db/collection.go +++ b/db/collection.go @@ -783,24 +783,29 @@ func (c *collection) create(ctx context.Context, txn datastore.Txn, doc *client. } func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { - colIndexKey := core.NewCollectionIndexKey(c.desc.Name, "user_name") - indexData, err := txn.Systemstore().Get(ctx, colIndexKey.ToDS()) - if err != nil { - return NewErrFailedToReadStoredIndexDesc(err) - } - var indexDesc client.IndexDescription - err = json.Unmarshal(indexData, &indexDesc) - if err != nil { - return NewErrInvalidStoredIndex(err) - } - fieldVal, err := doc.Get("name") - // if new doc doesn't have the index field, we don't need to index it - if err != nil { - return nil + indexes, err := c.db.getCollectionIndexes(ctx, txn, c.desc.Name) + err = err + for _, index := range indexes { + indexedFieldName := index.Fields[0].Name + fieldVal, err := doc.Get(indexedFieldName) + if err != nil { + return nil + } + colIndexKey := core.NewCollectionIndexKey(c.desc.Name, index.Name) + indexData, err := txn.Systemstore().Get(ctx, colIndexKey.ToDS()) + if err != nil { + return NewErrFailedToReadStoredIndexDesc(err) + } + var indexDesc client.IndexDescription + err = json.Unmarshal(indexData, &indexDesc) + if err != nil { + return NewErrInvalidStoredIndex(err) + } + colIndex := NewCollectionIndex(c, indexDesc) + docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) + return colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) } - colIndex := NewCollectionIndex(c, indexDesc) - docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) - return colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) + return nil } // Update an existing document with the new values. diff --git a/db/index.go b/db/index.go index 432ad4a7c4..cd138aef40 100644 --- a/db/index.go +++ b/db/index.go @@ -24,40 +24,58 @@ func NewCollectionIndex( collection client.Collection, desc client.IndexDescription, ) CollectionIndex { - return &collectionSimpleIndex{collection: collection, desc: desc} + index := &collectionSimpleIndex{collection: collection, desc: desc} + schema := collection.Description().Schema + fieldID := schema.GetFieldKey(desc.Fields[0].Name) + field := schema.Fields[fieldID] + if field.Kind == client.FieldKind_STRING { + index.convertFunc = func(val any) ([]byte, error) { + return []byte(val.(string)), nil + } + } else if field.Kind == client.FieldKind_INT { + index.convertFunc = func(val any) ([]byte, error) { + intVal := val.(int64) + return []byte(strconv.FormatInt(intVal, 10)), nil + } + } else { + panic("there is no test for this case") + } + return index } type collectionSimpleIndex struct { - collection client.Collection - desc client.IndexDescription + collection client.Collection + desc client.IndexDescription + convertFunc func(any) ([]byte, error) } var _ CollectionIndex = (*collectionSimpleIndex)(nil) -func (c *collectionSimpleIndex) Save( +func (i *collectionSimpleIndex) Save( ctx context.Context, txn datastore.Txn, key core.DataStoreKey, val any, ) error { - data := val.(string) + data, err := i.convertFunc(val) + err = err indexDataStoreKey := core.IndexDataStoreKey{} - indexDataStoreKey.CollectionID = strconv.Itoa(int(c.collection.ID())) + indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = "1" - indexDataStoreKey.FieldValues = []string{data, key.DocKey} - err := txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) + indexDataStoreKey.FieldValues = []string{string(data), key.DocKey} + err = txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) if err != nil { return NewErrFailedToStoreIndexedField("name", err) } return nil } -func (c *collectionSimpleIndex) Name() string { - return c.desc.Name +func (i *collectionSimpleIndex) Name() string { + return i.desc.Name } -func (c *collectionSimpleIndex) Description() client.IndexDescription { - return c.desc +func (i *collectionSimpleIndex) Description() client.IndexDescription { + return i.desc } func validateIndexDescriptionFields(fields []client.IndexedFieldDescription) error { diff --git a/db/index_test.go b/db/index_test.go index 21c8782fcd..dd2ce3efcb 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -30,6 +30,7 @@ const ( productsColName = "Products" testUsersColIndexName = "user_name" + testUsersColIndexAge = "user_age" ) type indexTestFixture struct { @@ -125,6 +126,15 @@ func getUsersIndexDescOnName() client.IndexDescription { } } +func getUsersIndexDescOnAge() client.IndexDescription { + return client.IndexDescription{ + Name: testUsersColIndexAge, + Fields: []client.IndexedFieldDescription{ + {Name: "age", Direction: client.Ascending}, + }, + } +} + func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescription { newDesc, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnName()) require.NoError(f.t, err) @@ -132,6 +142,13 @@ func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescrip return newDesc } +func (f *indexTestFixture) createUserCollectionIndexOnAge() client.IndexDescription { + newDesc, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnAge()) + require.NoError(f.t, err) + f.commitTxn() + return newDesc +} + func (f *indexTestFixture) dropIndex(colName, indexName string) error { return f.db.dropCollectionIndex(f.ctx, f.txn, colName, indexName) } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 36a1841b69..c98675f87e 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -3,6 +3,7 @@ package db import ( "encoding/json" "errors" + "fmt" "strconv" "testing" @@ -38,14 +39,10 @@ func (f *indexTestFixture) newUserDoc(name string, age int) *client.Document { return doc } -func (f *indexTestFixture) getNonUniqueIndexKey(fieldName string) core.IndexDataStoreKey { - colDesc := f.users.Description() - field, ok := colDesc.GetField(fieldName) - require.True(f.t, ok) - +func (f *indexTestFixture) getNonUniqueIndexKey() core.IndexDataStoreKey { key := core.IndexDataStoreKey{ CollectionID: strconv.Itoa(int(f.users.ID())), - IndexID: strconv.Itoa(int(field.ID)), + IndexID: strconv.Itoa(1), } return key } @@ -54,12 +51,11 @@ func (f *indexTestFixture) getNonUniqueDocIndexKey( doc *client.Document, fieldName string, ) core.IndexDataStoreKey { - key := f.getNonUniqueIndexKey(fieldName) + key := f.getNonUniqueIndexKey() fieldVal, err := doc.Get(fieldName) require.NoError(f.t, err) - fieldStrVal, ok := fieldVal.(string) - require.True(f.t, ok) + fieldStrVal := fmt.Sprintf("%v", fieldVal) key.FieldValues = []string{fieldStrVal, doc.Key().String()} @@ -81,14 +77,23 @@ func (f *indexTestFixture) getPrefixFromDataStore(prefix string) [][]byte { func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { mockedTxn := mocks.NewTxnWithMultistore(f.t) - indexDesc := getUsersIndexDescOnName() - indexDescData, err := json.Marshal(indexDesc) + indexOnNameDescData, err := json.Marshal(getUsersIndexDescOnName()) require.NoError(f.t, err) systemStoreOn := mockedTxn.MockSystemstore.EXPECT() + + colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, "") + matchPrefixFunc := func(q query.Query) bool { return q.Prefix == colIndexKey.ToDS().String() } + + systemStoreOn.Query(mock.Anything, mock.Anything).Unset() + systemStoreOn.Query(mock.Anything, mock.MatchedBy(matchPrefixFunc)).Maybe(). + Return(mocks.NewQueryResultsWithValues(f.t, indexOnNameDescData), nil) + systemStoreOn.Query(mock.Anything, mock.Anything).Maybe(). + Return(mocks.NewQueryResultsWithValues(f.t), nil) + systemStoreOn.Get(mock.Anything, mock.Anything).Unset() - colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, indexDesc.Name) - systemStoreOn.Get(mock.Anything, colIndexKey.ToDS()).Return(indexDescData, nil) + colIndexOnNameKey := core.NewCollectionIndexKey(f.users.Description().Name, testUsersColIndexName) + systemStoreOn.Get(mock.Anything, colIndexOnNameKey.ToDS()).Return(indexOnNameDescData, nil) systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) f.txn = mockedTxn @@ -143,7 +148,7 @@ func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { err = f.users.Create(f.ctx, doc) require.NoError(f.t, err) - key := f.getNonUniqueIndexKey("name") + key := f.getNonUniqueIndexKey() prefixes := f.getPrefixFromDataStore(key.ToString()) assert.Len(t, prefixes, 0) } @@ -167,17 +172,31 @@ func TestNonUnique_IfSystemStorageHasInvalidIndexDescription_Error(t *testing.T) func TestNonUnique_IfSystemStorageFailsToReadIndexDesc_Error(t *testing.T) { f := newIndexTestFixture(t) - indexDesc := f.createUserCollectionIndexOnName() + f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21) mockTxn := f.mockTxn() systemStoreOn := mockTxn.MockSystemstore.EXPECT() systemStoreOn.Get(mock.Anything, mock.Anything).Unset() - colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, indexDesc.Name) + colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, testUsersColIndexName) systemStoreOn.Get(mock.Anything, colIndexKey.ToDS()).Return([]byte{}, errors.New("error")) systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) require.ErrorIs(t, err, NewErrFailedToReadStoredIndexDesc(nil)) } + +func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnAge() + + doc := f.newUserDoc("John", 21) + f.saveToUsers(doc) + + key := f.getNonUniqueDocIndexKey(doc, "age") + + data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} From d569f2d026f6d141ff02c99b41fed131b9b10893 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 12 May 2023 11:04:12 +0200 Subject: [PATCH 035/120] Handle case sensitivity --- db/index.go | 9 ++++-- db/index_test.go | 82 +++++++++++++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 35 deletions(-) diff --git a/db/index.go b/db/index.go index cd138aef40..233f942839 100644 --- a/db/index.go +++ b/db/index.go @@ -37,6 +37,8 @@ func NewCollectionIndex( intVal := val.(int64) return []byte(strconv.FormatInt(intVal, 10)), nil } + } else if field.Kind == client.FieldKind_FLOAT { + // TODO: test } else { panic("there is no test for this case") } @@ -60,7 +62,8 @@ func (i *collectionSimpleIndex) Save( data, err := i.convertFunc(val) err = err indexDataStoreKey := core.IndexDataStoreKey{} - indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) + //indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) + indexDataStoreKey.CollectionID = strconv.Itoa(int(1)) indexDataStoreKey.IndexID = "1" indexDataStoreKey.FieldValues = []string{string(data), key.DocKey} err = txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) @@ -102,9 +105,9 @@ func generateIndexName(col client.Collection, fields []client.IndexedFieldDescri //if fields[0].Direction == client.Descending { //direction = "DESC" //} - sb.WriteString(strings.ToLower(col.Name())) + sb.WriteString(col.Name()) sb.WriteByte('_') - sb.WriteString(strings.ToLower(fields[0].Name)) + sb.WriteString(fields[0].Name) sb.WriteByte('_') sb.WriteString(direction) if inc > 1 { diff --git a/db/index_test.go b/db/index_test.go index dd2ce3efcb..3e770f6ff9 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -29,6 +29,13 @@ const ( usersColName = "Users" productsColName = "Products" + usersNameFieldName = "name" + usersAgeFieldName = "age" + usersWeightFieldName = "weight" + + productsPriceFieldName = "price" + productsDescFieldName = "description" + testUsersColIndexName = "user_name" testUsersColIndexAge = "user_age" ) @@ -51,17 +58,17 @@ func getUsersCollectionDesc() client.CollectionDescription { Kind: client.FieldKind_DocKey, }, { - Name: "name", + Name: usersNameFieldName, Kind: client.FieldKind_STRING, Typ: client.LWW_REGISTER, }, { - Name: "age", + Name: usersAgeFieldName, Kind: client.FieldKind_INT, Typ: client.LWW_REGISTER, }, { - Name: "weight", + Name: usersWeightFieldName, Kind: client.FieldKind_FLOAT, Typ: client.LWW_REGISTER, }, @@ -80,12 +87,12 @@ func getProductsCollectionDesc() client.CollectionDescription { Kind: client.FieldKind_DocKey, }, { - Name: "price", + Name: productsPriceFieldName, Kind: client.FieldKind_FLOAT, Typ: client.LWW_REGISTER, }, { - Name: "description", + Name: productsDescFieldName, Kind: client.FieldKind_STRING, Typ: client.LWW_REGISTER, }, @@ -94,19 +101,23 @@ func getProductsCollectionDesc() client.CollectionDescription { } } -func newIndexTestFixture(t *testing.T) *indexTestFixture { +func newIndexTestFixtureBare(t *testing.T) *indexTestFixture { ctx := context.Background() db, err := newMemoryDB(ctx) require.NoError(t, err) txn, err := db.NewTxn(ctx, false) require.NoError(t, err) - f := &indexTestFixture{ + return &indexTestFixture{ ctx: ctx, db: db, txn: txn, t: t, } +} + +func newIndexTestFixture(t *testing.T) *indexTestFixture { + f := newIndexTestFixtureBare(t) f.users = f.createCollection(getUsersCollectionDesc()) return f } @@ -121,7 +132,7 @@ func getUsersIndexDescOnName() client.IndexDescription { return client.IndexDescription{ Name: testUsersColIndexName, Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, + {Name: usersNameFieldName, Direction: client.Ascending}, }, } } @@ -130,7 +141,7 @@ func getUsersIndexDescOnAge() client.IndexDescription { return client.IndexDescription{ Name: testUsersColIndexAge, Fields: []client.IndexedFieldDescription{ - {Name: "age", Direction: client.Ascending}, + {Name: usersAgeFieldName, Direction: client.Ascending}, }, } } @@ -235,7 +246,7 @@ func TestCreateIndex_IfValidInput_CreateIndex(t *testing.T) { desc := client.IndexDescription{ Name: "some_index_name", Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, + {Name: usersNameFieldName, Direction: client.Ascending}, }, } resultDesc, err := f.createCollectionIndex(desc) @@ -262,7 +273,7 @@ func TestCreateIndex_IfFieldHasNoDirection_DefaultToAsc(t *testing.T) { desc := client.IndexDescription{ Name: "some_index_name", - Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, } newDesc, err := f.createCollectionIndex(desc) assert.NoError(t, err) @@ -270,18 +281,25 @@ func TestCreateIndex_IfFieldHasNoDirection_DefaultToAsc(t *testing.T) { } func TestCreateIndex_IfNameIsNotSpecified_GenerateWithLowerCase(t *testing.T) { - f := newIndexTestFixture(t) + f := newIndexTestFixtureBare(t) + colDesc := getUsersCollectionDesc() + const colName = "UsErS" + const fieldName = "NaMe" + colDesc.Name = colName + colDesc.Schema.Name = colName // Which one should we use? + colDesc.Schema.Fields[1].Name = fieldName + f.users = f.createCollection(colDesc) desc := client.IndexDescription{ Name: "", Fields: []client.IndexedFieldDescription{ - {Name: "Name", Direction: client.Ascending}, + {Name: fieldName, Direction: client.Ascending}, }, } - f.users.Description().Schema.Fields[1].Name = "Name" + newDesc, err := f.createCollectionIndex(desc) assert.NoError(t, err) - assert.Equal(t, newDesc.Name, "users_name_ASC") + assert.Equal(t, newDesc.Name, colName+"_"+fieldName+"_ASC") } func TestCreateIndex_IfSingleFieldInDescOrder_ReturnError(t *testing.T) { @@ -289,7 +307,7 @@ func TestCreateIndex_IfSingleFieldInDescOrder_ReturnError(t *testing.T) { desc := client.IndexDescription{ Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Descending}, + {Name: usersNameFieldName, Direction: client.Descending}, }, } _, err := f.createCollectionIndex(desc) @@ -302,11 +320,11 @@ func TestCreateIndex_IfIndexWithNameAlreadyExists_ReturnError(t *testing.T) { name := "some_index_name" desc1 := client.IndexDescription{ Name: name, - Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, } desc2 := client.IndexDescription{ Name: name, - Fields: []client.IndexedFieldDescription{{Name: "age"}}, + Fields: []client.IndexedFieldDescription{{Name: usersAgeFieldName}}, } _, err := f.createCollectionIndex(desc1) assert.NoError(t, err) @@ -317,18 +335,18 @@ func TestCreateIndex_IfIndexWithNameAlreadyExists_ReturnError(t *testing.T) { func TestCreateIndex_IfGeneratedNameMatchesExisting_AddIncrement(t *testing.T) { f := newIndexTestFixture(t) - name := "users_age_ASC" + name := usersColName + "_" + usersAgeFieldName + "_ASC" desc1 := client.IndexDescription{ Name: name, - Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, } desc2 := client.IndexDescription{ Name: name + "_2", - Fields: []client.IndexedFieldDescription{{Name: "weight"}}, + Fields: []client.IndexedFieldDescription{{Name: usersWeightFieldName}}, } desc3 := client.IndexDescription{ Name: "", - Fields: []client.IndexedFieldDescription{{Name: "age"}}, + Fields: []client.IndexedFieldDescription{{Name: usersAgeFieldName}}, } _, err := f.createCollectionIndex(desc1) assert.NoError(t, err) @@ -345,7 +363,7 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { name := "users_age_ASC" desc := client.IndexDescription{ Name: name, - Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, } _, err := f.createCollectionIndex(desc) assert.NoError(t, err) @@ -365,7 +383,7 @@ func TestCreateIndex_IfStorageFails_ReturnError(t *testing.T) { name := "users_age_ASC" desc := client.IndexDescription{ Name: name, - Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, } f.db.Close(f.ctx) @@ -378,7 +396,7 @@ func TestCreateIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { f := newIndexTestFixture(t) desc := client.IndexDescription{ - Fields: []client.IndexedFieldDescription{{Name: "price"}}, + Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, } _, err := f.createCollectionIndexFor(productsColName, desc) @@ -402,7 +420,7 @@ func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { usersIndexDesc := client.IndexDescription{ Name: "users_name_index", - Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, } _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) assert.NoError(t, err) @@ -410,7 +428,7 @@ func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { f.createCollection(getProductsCollectionDesc()) productsIndexDesc := client.IndexDescription{ Name: "products_description_index", - Fields: []client.IndexedFieldDescription{{Name: "price"}}, + Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, } _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) assert.NoError(t, err) @@ -448,7 +466,7 @@ func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { desc := client.IndexDescription{ Name: "some_index_name", Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, + {Name: usersNameFieldName, Direction: client.Ascending}, }, } descData, _ := json.Marshal(desc) @@ -464,7 +482,7 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) usersIndexDesc := client.IndexDescription{ Name: "users_name_index", - Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, } _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) assert.NoError(t, err) @@ -472,7 +490,7 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) f.createCollection(getProductsCollectionDesc()) productsIndexDesc := client.IndexDescription{ Name: "products_description_index", - Fields: []client.IndexedFieldDescription{{Name: "price"}}, + Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, } _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) assert.NoError(t, err) @@ -542,14 +560,14 @@ func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { f := newIndexTestFixture(t) _, err := f.createCollectionIndexFor(usersColName, client.IndexDescription{ Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, + {Name: usersNameFieldName, Direction: client.Ascending}, }, }) assert.NoError(f.t, err) _, err = f.createCollectionIndexFor(usersColName, client.IndexDescription{ Fields: []client.IndexedFieldDescription{ - {Name: "age", Direction: client.Ascending}, + {Name: usersAgeFieldName, Direction: client.Ascending}, }, }) assert.NoError(f.t, err) From 05e13aa5305ce49cfec4b8d67debeb38418710fb Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 12 May 2023 12:27:00 +0200 Subject: [PATCH 036/120] Save index with right collection ID --- db/index.go | 3 +- db/index_test.go | 38 +++++++++++++++----- db/indexed_docs_test.go | 77 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 100 insertions(+), 18 deletions(-) diff --git a/db/index.go b/db/index.go index 233f942839..002676f338 100644 --- a/db/index.go +++ b/db/index.go @@ -62,8 +62,7 @@ func (i *collectionSimpleIndex) Save( data, err := i.convertFunc(val) err = err indexDataStoreKey := core.IndexDataStoreKey{} - //indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) - indexDataStoreKey.CollectionID = strconv.Itoa(int(1)) + indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = "1" indexDataStoreKey.FieldValues = []string{string(data), key.DocKey} err = txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) diff --git a/db/index_test.go b/db/index_test.go index 3e770f6ff9..0c199a8ba1 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -33,19 +33,22 @@ const ( usersAgeFieldName = "age" usersWeightFieldName = "weight" - productsPriceFieldName = "price" - productsDescFieldName = "description" + productsIDFieldName = "id" + productsPriceFieldName = "price" + productsCategoryFieldName = "category" + productsAvailableFieldName = "available" testUsersColIndexName = "user_name" testUsersColIndexAge = "user_age" ) type indexTestFixture struct { - ctx context.Context - db *implicitTxnDB - txn datastore.Txn - users client.Collection - t *testing.T + ctx context.Context + db *implicitTxnDB + txn datastore.Txn + users client.Collection + products client.Collection + t *testing.T } func getUsersCollectionDesc() client.CollectionDescription { @@ -86,16 +89,26 @@ func getProductsCollectionDesc() client.CollectionDescription { Name: "_key", Kind: client.FieldKind_DocKey, }, + { + Name: productsIDFieldName, + Kind: client.FieldKind_INT, + Typ: client.LWW_REGISTER, + }, { Name: productsPriceFieldName, Kind: client.FieldKind_FLOAT, Typ: client.LWW_REGISTER, }, { - Name: productsDescFieldName, + Name: productsCategoryFieldName, Kind: client.FieldKind_STRING, Typ: client.LWW_REGISTER, }, + { + Name: productsAvailableFieldName, + Kind: client.FieldKind_BOOL, + Typ: client.LWW_REGISTER, + }, }, }, } @@ -146,6 +159,15 @@ func getUsersIndexDescOnAge() client.IndexDescription { } } +func getProductsIndexDescOnCategory() client.IndexDescription { + return client.IndexDescription{ + Name: testUsersColIndexAge, + Fields: []client.IndexedFieldDescription{ + {Name: productsCategoryFieldName, Direction: client.Ascending}, + }, + } +} + func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescription { newDesc, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnName()) require.NoError(f.t, err) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index c98675f87e..f26b491ea5 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -22,6 +22,13 @@ type userDoc struct { Weight float64 `json:"weight"` } +type productDoc struct { + ID int `json:"id"` + Price float64 `json:"price"` + Category string `json:"category"` + Available bool `json:"available"` +} + func (f *indexTestFixture) saveToUsers(doc *client.Document) { err := f.users.Create(f.ctx, doc) require.NoError(f.t, err) @@ -39,9 +46,32 @@ func (f *indexTestFixture) newUserDoc(name string, age int) *client.Document { return doc } -func (f *indexTestFixture) getNonUniqueIndexKey() core.IndexDataStoreKey { +func (f *indexTestFixture) newProdDoc(id int, price float64, cat string) *client.Document { + d := productDoc{ID: id, Price: price, Category: cat} + data, err := json.Marshal(d) + require.NoError(f.t, err) + + doc, err := client.NewDocFromJSON(data) + require.NoError(f.t, err) + return doc +} + +func (f *indexTestFixture) getNonUniqueIndexKey(colName string) core.IndexDataStoreKey { + cols, err := f.db.getAllCollections(f.ctx, f.txn) + require.NoError(f.t, err) + colID := -1 + for _, col := range cols { + if col.Name() == colName { + colID = int(col.ID()) + break + } + } + if colID == -1 { + panic(errors.New("collection not found")) + } + key := core.IndexDataStoreKey{ - CollectionID: strconv.Itoa(int(f.users.ID())), + CollectionID: strconv.Itoa(colID), IndexID: strconv.Itoa(1), } return key @@ -49,9 +79,9 @@ func (f *indexTestFixture) getNonUniqueIndexKey() core.IndexDataStoreKey { func (f *indexTestFixture) getNonUniqueDocIndexKey( doc *client.Document, - fieldName string, + colName, fieldName string, ) core.IndexDataStoreKey { - key := f.getNonUniqueIndexKey() + key := f.getNonUniqueIndexKey(colName) fieldVal, err := doc.Get(fieldName) require.NoError(f.t, err) @@ -107,7 +137,7 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveToUsers(doc) - key := f.getNonUniqueDocIndexKey(doc, "name") + key := f.getNonUniqueDocIndexKey(doc, usersColName, usersNameFieldName) data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -119,7 +149,7 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21) - key := f.getNonUniqueDocIndexKey(doc, "name") + key := f.getNonUniqueDocIndexKey(doc, usersColName, usersNameFieldName) mockTxn := f.mockTxn() @@ -148,7 +178,7 @@ func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { err = f.users.Create(f.ctx, doc) require.NoError(f.t, err) - key := f.getNonUniqueIndexKey() + key := f.getNonUniqueIndexKey(usersColName) prefixes := f.getPrefixFromDataStore(key.ToString()) assert.Len(t, prefixes, 0) } @@ -194,9 +224,40 @@ func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveToUsers(doc) - key := f.getNonUniqueDocIndexKey(doc, "age") + key := f.getNonUniqueDocIndexKey(doc, usersColName, usersAgeFieldName) data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) assert.Len(t, data, 0) } + +func TestNonUnique_IfMultipleCollectionsWithIndexes_StoreIndexWithCollectionID(t *testing.T) { + f := newIndexTestFixtureBare(t) + users := f.createCollection(getUsersCollectionDesc()) + products := f.createCollection(getProductsCollectionDesc()) + + _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) + require.NoError(f.t, err) + _, err = f.createCollectionIndexFor(products.Name(), getProductsIndexDescOnCategory()) + require.NoError(f.t, err) + f.commitTxn() + + userDoc := f.newUserDoc("John", 21) + prodDoc := f.newProdDoc(1, 3, "games") + + err = users.Create(f.ctx, userDoc) + require.NoError(f.t, err) + err = products.Create(f.ctx, prodDoc) + require.NoError(f.t, err) + f.commitTxn() + + userDocKey := f.getNonUniqueDocIndexKey(userDoc, usersColName, usersNameFieldName) + prodDocKey := f.getNonUniqueDocIndexKey(prodDoc, productsColName, productsCategoryFieldName) + + data, err := f.txn.Datastore().Get(f.ctx, userDocKey.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) + data, err = f.txn.Datastore().Get(f.ctx, prodDocKey.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} From 21e305db7cba976ff21be3cce8e5cc872ea4f40d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 15 May 2023 09:59:35 +0200 Subject: [PATCH 037/120] Assign incremented id to a new index Implement indexKeyBuilder --- client/index.go | 1 + db/errors.go | 6 +++ db/index.go | 24 ++++++--- db/index_test.go | 82 +++++++++++++++++++++++------ db/indexed_docs_test.go | 114 +++++++++++++++++++++++++++++++--------- 5 files changed, 176 insertions(+), 51 deletions(-) diff --git a/client/index.go b/client/index.go index a7febb45c9..2608ae1466 100644 --- a/client/index.go +++ b/client/index.go @@ -14,6 +14,7 @@ type IndexedFieldDescription struct { type IndexDescription struct { Name string + ID uint32 Fields []IndexedFieldDescription Unique bool } diff --git a/db/errors.go b/db/errors.go index 0e6aa019ce..3ea830e8ac 100644 --- a/db/errors.go +++ b/db/errors.go @@ -38,6 +38,7 @@ const ( errDocumentAlreadyExists string = "a document with the given dockey already exists" errDocumentDeleted string = "a document with the given dockey has been deleted" errIndexMissingFields string = "index missing fields" + errNonZeroIndexIDProvided string = "non-zero index ID provided" errIndexFieldMissingName string = "index field missing name" errIndexFieldMissingDirection string = "index field missing direction" errIndexSingleFieldWrongDirection string = "wrong direction for index with a single field" @@ -148,6 +149,11 @@ func NewErrFailedToReadStoredIndexDesc(inner error) error { return errors.Wrap(errFailedToReadStoredIndexDesc, inner) } +// NewErrNonZeroIndexIDProvided returns a new error indicating that a non-zero index ID was provided. +func NewErrNonZeroIndexIDProvided(indexID uint32) error { + return errors.New(errNonZeroIndexIDProvided, errors.NewKV("ID", indexID)) +} + // NewErrFailedToGetCollection returns a new error indicating that the collection could not be obtained. func NewErrFailedToGetCollection(name string, inner error) error { return errors.Wrap(errFailedToGetCollection, inner, errors.NewKV("Name", name)) diff --git a/db/index.go b/db/index.go index 002676f338..3d64eee54a 100644 --- a/db/index.go +++ b/db/index.go @@ -3,6 +3,7 @@ package db import ( "context" "encoding/json" + "fmt" "strconv" "strings" @@ -80,19 +81,22 @@ func (i *collectionSimpleIndex) Description() client.IndexDescription { return i.desc } -func validateIndexDescriptionFields(fields []client.IndexedFieldDescription) error { - if len(fields) == 0 { +func validateIndexDescription(desc client.IndexDescription) error { + if desc.ID != 0 { + return NewErrNonZeroIndexIDProvided(desc.ID) + } + if len(desc.Fields) == 0 { return ErrIndexMissingFields } - if len(fields) == 1 && fields[0].Direction == client.Descending { + if len(desc.Fields) == 1 && desc.Fields[0].Direction == client.Descending { return ErrIndexSingleFieldWrongDirection } - for i := range fields { - if fields[i].Name == "" { + for i := range desc.Fields { + if desc.Fields[i].Name == "" { return ErrIndexFieldMissingName } - if fields[i].Direction == "" { - fields[i].Direction = client.Ascending + if desc.Fields[i].Direction == "" { + desc.Fields[i].Direction = client.Ascending } } return nil @@ -175,7 +179,7 @@ func (c *collection) createIndex( ctx context.Context, desc client.IndexDescription, ) (CollectionIndex, error) { - err := validateIndexDescriptionFields(desc.Fields) + err := validateIndexDescription(desc) if err != nil { return nil, err } @@ -200,6 +204,10 @@ func (c *collection) createIndex( return nil, err } + colSeq, err := c.db.getSequence(ctx, txn, fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, c.ID())) + colID, err := colSeq.next(ctx, txn) + desc.ID = uint32(colID) + err = txn.Systemstore().Put(ctx, indexKey.ToDS(), buf) if err != nil { return nil, err diff --git a/db/index_test.go b/db/index_test.go index 0c199a8ba1..099c88980e 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -12,7 +12,9 @@ package db import ( "context" + "encoding/binary" "encoding/json" + "fmt" "testing" ds "github.com/ipfs/go-datastore" @@ -262,6 +264,22 @@ func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { assert.EqualError(t, err, errIndexMissingFields) } +func TestCreateIndex_IfIndexDescriptionIDIsNotZero_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + for _, id := range []uint32{1, 20, 999} { + desc := client.IndexDescription{ + Name: "some_index_name", + ID: id, + Fields: []client.IndexedFieldDescription{ + {Name: usersNameFieldName, Direction: client.Ascending}, + }, + } + _, err := f.createCollectionIndex(desc) + assert.ErrorIs(t, err, NewErrNonZeroIndexIDProvided(0)) + } +} + func TestCreateIndex_IfValidInput_CreateIndex(t *testing.T) { f := newIndexTestFixture(t) @@ -273,8 +291,8 @@ func TestCreateIndex_IfValidInput_CreateIndex(t *testing.T) { } resultDesc, err := f.createCollectionIndex(desc) assert.NoError(t, err) - assert.Equal(t, resultDesc.Name, desc.Name) - assert.Equal(t, resultDesc, desc) + assert.Equal(t, desc.Name, resultDesc.Name) + assert.Equal(t, desc, resultDesc) } func TestCreateIndex_IfFieldNameIsEmpty_ReturnError(t *testing.T) { @@ -299,7 +317,7 @@ func TestCreateIndex_IfFieldHasNoDirection_DefaultToAsc(t *testing.T) { } newDesc, err := f.createCollectionIndex(desc) assert.NoError(t, err) - assert.Equal(t, newDesc.Fields[0].Direction, client.Ascending) + assert.Equal(t, client.Ascending, newDesc.Fields[0].Direction) } func TestCreateIndex_IfNameIsNotSpecified_GenerateWithLowerCase(t *testing.T) { @@ -321,7 +339,7 @@ func TestCreateIndex_IfNameIsNotSpecified_GenerateWithLowerCase(t *testing.T) { newDesc, err := f.createCollectionIndex(desc) assert.NoError(t, err) - assert.Equal(t, newDesc.Name, colName+"_"+fieldName+"_ASC") + assert.Equal(t, colName+"_"+fieldName+"_ASC", newDesc.Name) } func TestCreateIndex_IfSingleFieldInDescOrder_ReturnError(t *testing.T) { @@ -376,7 +394,7 @@ func TestCreateIndex_IfGeneratedNameMatchesExisting_AddIncrement(t *testing.T) { assert.NoError(t, err) newDesc3, err := f.createCollectionIndex(desc3) assert.NoError(t, err) - assert.Equal(t, newDesc3.Name, name+"_3") + assert.Equal(t, name+"_3", newDesc3.Name) } func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { @@ -396,7 +414,7 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { var deserialized client.IndexDescription err = json.Unmarshal(data, &deserialized) assert.NoError(t, err) - assert.Equal(t, deserialized, desc) + assert.Equal(t, desc, deserialized) } func TestCreateIndex_IfStorageFails_ReturnError(t *testing.T) { @@ -458,15 +476,15 @@ func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { indexes, err := f.getAllIndexes() assert.NoError(t, err) - require.Equal(t, len(indexes), 2) + require.Equal(t, 2, len(indexes)) usersIndexIndex := 0 if indexes[0].CollectionName != usersColName { usersIndexIndex = 1 } - assert.Equal(t, indexes[usersIndexIndex].Index, usersIndexDesc) - assert.Equal(t, indexes[usersIndexIndex].CollectionName, usersColName) - assert.Equal(t, indexes[1-usersIndexIndex].Index, productsIndexDesc) - assert.Equal(t, indexes[1-usersIndexIndex].CollectionName, productsColName) + assert.Equal(t, usersIndexDesc, indexes[usersIndexIndex].Index) + assert.Equal(t, usersColName, indexes[usersIndexIndex].CollectionName) + assert.Equal(t, productsIndexDesc, indexes[1-usersIndexIndex].Index) + assert.Equal(t, productsColName, indexes[1-usersIndexIndex].CollectionName) } func TestGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { @@ -519,13 +537,13 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) userIndexes, err := f.getCollectionIndexes(usersColName) assert.NoError(t, err) - require.Equal(t, len(userIndexes), 1) - assert.Equal(t, userIndexes[0], usersIndexDesc) + require.Equal(t, 1, len(userIndexes)) + assert.Equal(t, usersIndexDesc, userIndexes[0]) productIndexes, err := f.getCollectionIndexes(productsColName) assert.NoError(t, err) - require.Equal(t, len(productIndexes), 1) - assert.Equal(t, productIndexes[0], productsIndexDesc) + require.Equal(t, 1, len(productIndexes)) + assert.Equal(t, productsIndexDesc, productIndexes[0]) } func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { @@ -594,12 +612,12 @@ func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { }) assert.NoError(f.t, err) - assert.Equal(t, f.countIndexPrefixes(usersColName, ""), 2) + assert.Equal(t, 2, f.countIndexPrefixes(usersColName, "")) err = f.dropAllIndexes(usersColName) assert.NoError(t, err) - assert.Equal(t, f.countIndexPrefixes(usersColName, ""), 0) + assert.Equal(t, 0, f.countIndexPrefixes(usersColName, "")) } func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { @@ -611,3 +629,33 @@ func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { err := f.dropAllIndexes(usersColName) assert.Error(t, err) } + +func TestCreateIndex_WithMultipleCollectionsAndIndexes_AssignIncrementedIDPerCollection(t *testing.T) { + f := newIndexTestFixtureBare(t) + users := f.createCollection(getUsersCollectionDesc()) + products := f.createCollection(getProductsCollectionDesc()) + + makeIndex := func(fieldName string) client.IndexDescription { + return client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + {Name: fieldName, Direction: client.Ascending}, + }, + } + } + + createIndexAndAssert := func(col client.Collection, fieldName string, expectedID uint32) { + desc, err := f.createCollectionIndexFor(col.Name(), makeIndex(fieldName)) + require.NoError(t, err) + assert.Equal(t, expectedID, desc.ID) + seqKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, col.ID())) + storedSeqKey, err := f.txn.Systemstore().Get(f.ctx, seqKey.ToDS()) + assert.NoError(t, err) + storedSeqVal := binary.BigEndian.Uint64(storedSeqKey) + assert.Equal(t, expectedID, uint32(storedSeqVal)) + } + + createIndexAndAssert(users, usersNameFieldName, 1) + createIndexAndAssert(users, usersAgeFieldName, 2) + createIndexAndAssert(products, productsIDFieldName, 1) + createIndexAndAssert(products, productsCategoryFieldName, 2) +} diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index f26b491ea5..e98596e718 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -56,38 +56,81 @@ func (f *indexTestFixture) newProdDoc(id int, price float64, cat string) *client return doc } -func (f *indexTestFixture) getNonUniqueIndexKey(colName string) core.IndexDataStoreKey { - cols, err := f.db.getAllCollections(f.ctx, f.txn) - require.NoError(f.t, err) - colID := -1 +type indexKeyBuilder struct { + f *indexTestFixture + colName string + fieldName string + doc *client.Document + isUnique bool +} + +func newIndexKeyBuilder(f *indexTestFixture) *indexKeyBuilder { + return &indexKeyBuilder{f: f} +} + +func (b *indexKeyBuilder) Col(colName string) *indexKeyBuilder { + b.colName = colName + return b +} + +func (b *indexKeyBuilder) Field(fieldName string) *indexKeyBuilder { + b.fieldName = fieldName + return b +} + +func (b *indexKeyBuilder) Doc(doc *client.Document) *indexKeyBuilder { + b.doc = doc + return b +} + +func (b *indexKeyBuilder) Unique() *indexKeyBuilder { + b.isUnique = true + return b +} + +func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { + key := core.IndexDataStoreKey{} + + if b.colName == "" { + return key + } + + cols, err := b.f.db.getAllCollections(b.f.ctx, b.f.txn) + require.NoError(b.f.t, err) + var collection client.Collection for _, col := range cols { - if col.Name() == colName { - colID = int(col.ID()) + if col.Name() == b.colName { + collection = col break } } - if colID == -1 { + if collection == nil { panic(errors.New("collection not found")) } + key.CollectionID = strconv.Itoa(int(collection.ID())) - key := core.IndexDataStoreKey{ - CollectionID: strconv.Itoa(colID), - IndexID: strconv.Itoa(1), + if b.fieldName == "" { + return key + } + + indexes, err := collection.GetIndexes(b.f.ctx) + require.NoError(b.f.t, err) + for _, index := range indexes { + if index.Fields[0].Name == b.fieldName { + key.IndexID = strconv.Itoa(int(index.ID)) + break + } } - return key -} -func (f *indexTestFixture) getNonUniqueDocIndexKey( - doc *client.Document, - colName, fieldName string, -) core.IndexDataStoreKey { - key := f.getNonUniqueIndexKey(colName) + if b.doc == nil { + return key + } - fieldVal, err := doc.Get(fieldName) - require.NoError(f.t, err) + fieldVal, err := b.doc.Get(b.fieldName) + require.NoError(b.f.t, err) fieldStrVal := fmt.Sprintf("%v", fieldVal) - key.FieldValues = []string{fieldStrVal, doc.Key().String()} + key.FieldValues = []string{fieldStrVal, b.doc.Key().String()} return key } @@ -137,7 +180,7 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveToUsers(doc) - key := f.getNonUniqueDocIndexKey(doc, usersColName, usersNameFieldName) + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -149,7 +192,7 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21) - key := f.getNonUniqueDocIndexKey(doc, usersColName, usersNameFieldName) + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Build() mockTxn := f.mockTxn() @@ -178,7 +221,7 @@ func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { err = f.users.Create(f.ctx, doc) require.NoError(f.t, err) - key := f.getNonUniqueIndexKey(usersColName) + key := newIndexKeyBuilder(f).Col(usersColName).Build() prefixes := f.getPrefixFromDataStore(key.ToString()) assert.Len(t, prefixes, 0) } @@ -224,7 +267,7 @@ func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveToUsers(doc) - key := f.getNonUniqueDocIndexKey(doc, usersColName, usersAgeFieldName) + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -251,8 +294,8 @@ func TestNonUnique_IfMultipleCollectionsWithIndexes_StoreIndexWithCollectionID(t require.NoError(f.t, err) f.commitTxn() - userDocKey := f.getNonUniqueDocIndexKey(userDoc, usersColName, usersNameFieldName) - prodDocKey := f.getNonUniqueDocIndexKey(prodDoc, productsColName, productsCategoryFieldName) + userDocKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(userDoc).Build() + prodDocKey := newIndexKeyBuilder(f).Col(productsColName).Field(productsCategoryFieldName).Doc(prodDoc).Build() data, err := f.txn.Datastore().Get(f.ctx, userDocKey.ToDS()) require.NoError(t, err) @@ -261,3 +304,22 @@ func TestNonUnique_IfMultipleCollectionsWithIndexes_StoreIndexWithCollectionID(t require.NoError(t, err) assert.Len(t, data, 0) } + +func TestNonUnique_IfMultipleIndexes_StoreIndexWithIndexID(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + f.createUserCollectionIndexOnAge() + + doc := f.newUserDoc("John", 21) + f.saveToUsers(doc) + + nameKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + ageKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() + + data, err := f.txn.Datastore().Get(f.ctx, nameKey.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) + data, err = f.txn.Datastore().Get(f.ctx, ageKey.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} From 37ae5e971bf95b7fc071ce8452e7a1f9a5188d65 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 15 May 2023 13:06:20 +0200 Subject: [PATCH 038/120] Index a new doc with multiple indexes --- db/collection.go | 5 ++++- db/index.go | 45 ++++++++++++++++++++++++++++++++++------- db/index_test.go | 26 +++++++++++++++++++++--- db/indexed_docs_test.go | 11 ++++++---- 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/db/collection.go b/db/collection.go index d317488ed9..4abfc9b4f6 100644 --- a/db/collection.go +++ b/db/collection.go @@ -803,7 +803,10 @@ func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *cl } colIndex := NewCollectionIndex(c, indexDesc) docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) - return colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) + err = colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) + if err != nil { + return err + } } return nil } diff --git a/db/index.go b/db/index.go index 3d64eee54a..6d864fc4ab 100644 --- a/db/index.go +++ b/db/index.go @@ -64,7 +64,7 @@ func (i *collectionSimpleIndex) Save( err = err indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) - indexDataStoreKey.IndexID = "1" + indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) indexDataStoreKey.FieldValues = []string{string(data), key.DocKey} err = txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) if err != nil { @@ -172,7 +172,38 @@ func (c *collection) dropAllIndexes(ctx context.Context) error { } func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { - return nil, nil + prefix := core.NewCollectionIndexKey(c.Name(), "") + txn, err := c.getTxn(ctx, false) + if err != nil { + //return nil, err + } + q, err := txn.Systemstore().Query(ctx, query.Query{ + Prefix: prefix.ToString(), + }) + if err != nil { + //return nil, NewErrFailedToCreateCollectionQuery(err) + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + indexDescriptions := make([]client.IndexDescription, 0) + for res := range q.Next() { + if res.Error != nil { + //return nil, err + } + + var indexDesc client.IndexDescription + err = json.Unmarshal(res.Value, &indexDesc) + if err != nil { + //return nil, NewErrInvalidStoredIndex(err) + } + indexDescriptions = append(indexDescriptions, indexDesc) + } + + return indexDescriptions, nil } func (c *collection) createIndex( @@ -194,11 +225,6 @@ func (c *collection) createIndex( return nil, err } - buf, err := json.Marshal(desc) - if err != nil { - return nil, err - } - txn, err := c.getTxn(ctx, false) if err != nil { return nil, err @@ -208,6 +234,11 @@ func (c *collection) createIndex( colID, err := colSeq.next(ctx, txn) desc.ID = uint32(colID) + buf, err := json.Marshal(desc) + if err != nil { + return nil, err + } + err = txn.Systemstore().Put(ctx, indexKey.ToDS(), buf) if err != nil { return nil, err diff --git a/db/index_test.go b/db/index_test.go index 099c88980e..d607376912 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -292,7 +292,8 @@ func TestCreateIndex_IfValidInput_CreateIndex(t *testing.T) { resultDesc, err := f.createCollectionIndex(desc) assert.NoError(t, err) assert.Equal(t, desc.Name, resultDesc.Name) - assert.Equal(t, desc, resultDesc) + assert.Equal(t, desc.Fields, resultDesc.Fields) + assert.Equal(t, desc.Unique, resultDesc.Unique) } func TestCreateIndex_IfFieldNameIsEmpty_ReturnError(t *testing.T) { @@ -414,6 +415,7 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { var deserialized client.IndexDescription err = json.Unmarshal(data, &deserialized) assert.NoError(t, err) + desc.ID = 1 assert.Equal(t, desc, deserialized) } @@ -481,9 +483,9 @@ func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { if indexes[0].CollectionName != usersColName { usersIndexIndex = 1 } - assert.Equal(t, usersIndexDesc, indexes[usersIndexIndex].Index) + assert.Equal(t, usersIndexDesc.Name, indexes[usersIndexIndex].Index.Name) assert.Equal(t, usersColName, indexes[usersIndexIndex].CollectionName) - assert.Equal(t, productsIndexDesc, indexes[1-usersIndexIndex].Index) + assert.Equal(t, productsIndexDesc.Name, indexes[1-usersIndexIndex].Index.Name) assert.Equal(t, productsColName, indexes[1-usersIndexIndex].CollectionName) } @@ -538,11 +540,13 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) userIndexes, err := f.getCollectionIndexes(usersColName) assert.NoError(t, err) require.Equal(t, 1, len(userIndexes)) + usersIndexDesc.ID = 1 assert.Equal(t, usersIndexDesc, userIndexes[0]) productIndexes, err := f.getCollectionIndexes(productsColName) assert.NoError(t, err) require.Equal(t, 1, len(productIndexes)) + productsIndexDesc.ID = 1 assert.Equal(t, productsIndexDesc, productIndexes[0]) } @@ -567,6 +571,22 @@ func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) } +func TestCollectionGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + f.createUserCollectionIndexOnAge() + + indexes, err := f.users.GetIndexes(f.ctx) + assert.NoError(t, err) + require.Len(t, indexes, 2) + require.ElementsMatch(t, + []string{testUsersColIndexName, testUsersColIndexAge}, + []string{indexes[0].Name, indexes[1].Name}, + ) + require.ElementsMatch(t, []uint32{1, 2}, []uint32{indexes[0].ID, indexes[1].ID}) +} + func TestDropIndex_ShouldDeleteIndex(t *testing.T) { f := newIndexTestFixture(t) desc := f.createUserCollectionIndexOnName() diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index e98596e718..614435ae1b 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -150,7 +150,9 @@ func (f *indexTestFixture) getPrefixFromDataStore(prefix string) [][]byte { func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { mockedTxn := mocks.NewTxnWithMultistore(f.t) - indexOnNameDescData, err := json.Marshal(getUsersIndexDescOnName()) + desc := getUsersIndexDescOnName() + desc.ID = 1 + indexOnNameDescData, err := json.Marshal(desc) require.NoError(f.t, err) systemStoreOn := mockedTxn.MockSystemstore.EXPECT() @@ -179,8 +181,9 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveToUsers(doc) + //f.commitTxn() - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Build() + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -192,7 +195,7 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21) - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Build() + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() mockTxn := f.mockTxn() @@ -267,7 +270,7 @@ func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveToUsers(doc) - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Build() + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) From f4699d81c98ed341e18d43ee3693c394addc2278 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 16 May 2023 13:38:57 +0200 Subject: [PATCH 039/120] Store indexed fields of different types --- db/errors.go | 7 +++ db/index.go | 77 +++++++++++++++++++------ db/indexed_docs_test.go | 125 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 184 insertions(+), 25 deletions(-) diff --git a/db/errors.go b/db/errors.go index 3ea830e8ac..c78c7e204e 100644 --- a/db/errors.go +++ b/db/errors.go @@ -49,6 +49,7 @@ const ( errCollectionDoesntExisting string = "collection with given name doesn't exist" errFailedToStoreIndexedField string = "failed to store indexed field" errFailedToReadStoredIndexDesc string = "failed to read stored index description" + errCanNotIndexInvalidFieldValue string = "can not index invalid field value" ) var ( @@ -149,6 +150,12 @@ func NewErrFailedToReadStoredIndexDesc(inner error) error { return errors.Wrap(errFailedToReadStoredIndexDesc, inner) } +// NewErrCanNotIndexInvalidFieldValue returns a new error indicating that the field value is invalid +// and cannot be indexed. +func NewErrCanNotIndexInvalidFieldValue(inner error) error { + return errors.Wrap(errCanNotIndexInvalidFieldValue, inner) +} + // NewErrNonZeroIndexIDProvided returns a new error indicating that a non-zero index ID was provided. func NewErrNonZeroIndexIDProvided(indexID uint32) error { return errors.New(errNonZeroIndexIDProvided, errors.NewKV("ID", indexID)) diff --git a/db/index.go b/db/index.go index 6d864fc4ab..f3cc8b5c74 100644 --- a/db/index.go +++ b/db/index.go @@ -6,6 +6,7 @@ import ( "fmt" "strconv" "strings" + "time" ds "github.com/ipfs/go-datastore" @@ -13,6 +14,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/errors" ) type CollectionIndex interface { @@ -21,6 +23,57 @@ type CollectionIndex interface { Description() client.IndexDescription } +func getFieldValConverter(kind client.FieldKind) func(any) ([]byte, error) { + switch kind { + case client.FieldKind_STRING: + return func(val any) ([]byte, error) { + return []byte(val.(string)), nil + } + case client.FieldKind_INT: + return func(val any) ([]byte, error) { + intVal, ok := val.(int64) + if !ok { + return nil, errors.New("invalid int value") + } + return []byte(strconv.FormatInt(intVal, 10)), nil + } + case client.FieldKind_FLOAT: + return func(val any) ([]byte, error) { + floatVal, ok := val.(float64) + if !ok { + return nil, errors.New("invalid float value") + } + return []byte(strconv.FormatFloat(floatVal, 'f', -1, 64)), nil + } + case client.FieldKind_BOOL: + return func(val any) ([]byte, error) { + boolVal, ok := val.(bool) + if !ok { + return nil, errors.New("invalid bool value") + } + var intVal int64 = 0 + if boolVal { + intVal = 1 + } + return []byte(strconv.FormatInt(intVal, 10)), nil + } + case client.FieldKind_DATETIME: + return func(val any) ([]byte, error) { + timeStrVal, ok := val.(string) + if !ok { + return nil, errors.New("invalid datetime value") + } + _, err := time.Parse(time.RFC3339, timeStrVal) + if err != nil { + return nil, err + } + return []byte(timeStrVal), nil + } + default: + panic("there is no test for this case") + } +} + func NewCollectionIndex( collection client.Collection, desc client.IndexDescription, @@ -29,20 +82,7 @@ func NewCollectionIndex( schema := collection.Description().Schema fieldID := schema.GetFieldKey(desc.Fields[0].Name) field := schema.Fields[fieldID] - if field.Kind == client.FieldKind_STRING { - index.convertFunc = func(val any) ([]byte, error) { - return []byte(val.(string)), nil - } - } else if field.Kind == client.FieldKind_INT { - index.convertFunc = func(val any) ([]byte, error) { - intVal := val.(int64) - return []byte(strconv.FormatInt(intVal, 10)), nil - } - } else if field.Kind == client.FieldKind_FLOAT { - // TODO: test - } else { - panic("there is no test for this case") - } + index.convertFunc = getFieldValConverter(field.Kind) return index } @@ -61,14 +101,17 @@ func (i *collectionSimpleIndex) Save( val any, ) error { data, err := i.convertFunc(val) - err = err + if err != nil { + return NewErrCanNotIndexInvalidFieldValue(err) + } indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) indexDataStoreKey.FieldValues = []string{string(data), key.DocKey} - err = txn.Datastore().Put(ctx, indexDataStoreKey.ToDS(), []byte{}) + keyStr := indexDataStoreKey.ToDS() + err = txn.Datastore().Put(ctx, keyStr, []byte{}) if err != nil { - return NewErrFailedToStoreIndexedField("name", err) + return NewErrFailedToStoreIndexedField(indexDataStoreKey.IndexID, err) } return nil } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 614435ae1b..cae8335e9d 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -6,6 +6,7 @@ import ( "fmt" "strconv" "testing" + "time" "github.com/ipfs/go-datastore/query" "github.com/sourcenetwork/defradb/client" @@ -56,11 +57,15 @@ func (f *indexTestFixture) newProdDoc(id int, price float64, cat string) *client return doc } +// indexKeyBuilder is a helper for building index keys that can be turned into a string. +// The format of the non-unique index key is: "////" +// Example: "/5/1/12/bae-61cd6879-63ca-5ca9-8731-470a3c1dac69" type indexKeyBuilder struct { f *indexTestFixture colName string fieldName string doc *client.Document + values []string isUnique bool } @@ -73,16 +78,30 @@ func (b *indexKeyBuilder) Col(colName string) *indexKeyBuilder { return b } +// Field sets the field name for the index key. +// If the field name is not set, the index key will contain only collection id. +// When building a key it will it will find the field id to use in the key. func (b *indexKeyBuilder) Field(fieldName string) *indexKeyBuilder { b.fieldName = fieldName return b } +// Doc sets the document for the index key. +// For non-unique index keys, it will try to find the field value in the document +// corresponding to the field name set in the builder. +// As the last value in the index key, it will use the document id. func (b *indexKeyBuilder) Doc(doc *client.Document) *indexKeyBuilder { b.doc = doc return b } +// Values sets the values for the index key. +// It will override the field values stored in the document. +func (b *indexKeyBuilder) Values(values ...string) *indexKeyBuilder { + b.values = values + return b +} + func (b *indexKeyBuilder) Unique() *indexKeyBuilder { b.isUnique = true return b @@ -122,15 +141,20 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { } } - if b.doc == nil { - return key - } - - fieldVal, err := b.doc.Get(b.fieldName) - require.NoError(b.f.t, err) - fieldStrVal := fmt.Sprintf("%v", fieldVal) + if b.doc != nil { + var fieldStrVal string + if len(b.values) == 0 { + fieldVal, err := b.doc.Get(b.fieldName) + require.NoError(b.f.t, err) + fieldStrVal = fmt.Sprintf("%v", fieldVal) + } else { + fieldStrVal = b.values[0] + } - key.FieldValues = []string{fieldStrVal, b.doc.Key().String()} + key.FieldValues = []string{fieldStrVal, b.doc.Key().String()} + } else if len(b.values) > 0 { + key.FieldValues = b.values + } return key } @@ -326,3 +350,88 @@ func TestNonUnique_IfMultipleIndexes_StoreIndexWithIndexID(t *testing.T) { require.NoError(t, err) assert.Len(t, data, 0) } + +func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { + f := newIndexTestFixtureBare(t) + + now := time.Now() + nowStr := now.Format(time.RFC3339) + + testCase := []struct { + Name string + FieldKind client.FieldKind + FieldVal any + ShouldFail bool + Stored string + }{ + {Name: "invalid int", FieldKind: client.FieldKind_INT, FieldVal: "invalid", ShouldFail: true}, + {Name: "invalid float", FieldKind: client.FieldKind_FLOAT, FieldVal: "invalid", ShouldFail: true}, + {Name: "invalid bool", FieldKind: client.FieldKind_BOOL, FieldVal: "invalid", ShouldFail: true}, + {Name: "invalid datetime", FieldKind: client.FieldKind_DATETIME, FieldVal: nowStr[1:], ShouldFail: true}, + + {Name: "valid int", FieldKind: client.FieldKind_INT, FieldVal: 12, Stored: "12"}, + {Name: "valid float", FieldKind: client.FieldKind_FLOAT, FieldVal: 36.654, Stored: "36.654"}, + {Name: "valid bool true", FieldKind: client.FieldKind_BOOL, FieldVal: true, Stored: "1"}, + {Name: "valid bool false", FieldKind: client.FieldKind_BOOL, FieldVal: false, Stored: "0"}, + {Name: "valid datetime string", FieldKind: client.FieldKind_DATETIME, FieldVal: nowStr, Stored: nowStr}, + } + + for i, tc := range testCase { + desc := client.CollectionDescription{ + Name: "testTypeCol" + strconv.Itoa(i), + Schema: client.SchemaDescription{ + Fields: []client.FieldDescription{ + { + Name: "_key", + Kind: client.FieldKind_DocKey, + }, + { + Name: "field", + Kind: tc.FieldKind, + Typ: client.LWW_REGISTER, + }, + }, + }, + } + + collection := f.createCollection(desc) + + indexDesc := client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + {Name: "field", Direction: client.Ascending}, + }, + } + + _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + require.NoError(f.t, err) + f.commitTxn() + + d := struct { + Field any `json:"field"` + }{Field: tc.FieldVal} + data, err := json.Marshal(d) + require.NoError(f.t, err) + doc, err := client.NewDocFromJSON(data) + require.NoError(f.t, err) + + err = collection.Create(f.ctx, doc) + f.commitTxn() + if tc.ShouldFail { + require.ErrorIs(f.t, err, NewErrCanNotIndexInvalidFieldValue(nil), "test case: %s", tc.Name) + } else { + assertMsg := fmt.Sprintf("test case: %s", tc.Name) + require.NoError(f.t, err, assertMsg) + + keyBuilder := newIndexKeyBuilder(f).Col(collection.Name()).Field("field").Doc(doc) + if tc.Stored != "" { + keyBuilder.Values(tc.Stored) + } + key := keyBuilder.Build() + + keyStr := key.ToDS() + data, err := f.txn.Datastore().Get(f.ctx, keyStr) + require.NoError(t, err, assertMsg) + assert.Len(t, data, 0, assertMsg) + } + } +} From 39e19fdf312a4303bba1634484dff12d0519743f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 17 May 2023 12:37:03 +0200 Subject: [PATCH 040/120] Implement GetIndexes method --- datastore/mocks/utils.go | 18 +++++-- db/collection.go | 15 ++++-- db/index.go | 32 ++++++++--- db/index_test.go | 114 +++++++++++++++++++++++++++++++++++++++ db/indexed_docs_test.go | 4 +- 5 files changed, 165 insertions(+), 18 deletions(-) diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go index a4875fb22e..e12e5dc077 100644 --- a/datastore/mocks/utils.go +++ b/datastore/mocks/utils.go @@ -56,7 +56,7 @@ func prepareDAGStore(t *testing.T) *DAGStore { func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { txn := NewTxn(t) - txn.EXPECT().OnSuccess(mock.Anything) + txn.EXPECT().OnSuccess(mock.Anything).Maybe() result := &MultiStoreTxn{ Txn: txn, @@ -76,11 +76,19 @@ func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { return result } -func NewQueryResultsWithValues(t *testing.T, entries ...[]byte) *Results { +func NewQueryResultsWithValues(t *testing.T, values ...[]byte) *Results { + results := make([]query.Result, len(values)) + for i, value := range values { + results[i] = query.Result{Entry: query.Entry{Value: value}} + } + return NewQueryResultsWithResults(t, results...) +} + +func NewQueryResultsWithResults(t *testing.T, results ...query.Result) *Results { queryResults := NewResults(t) - resultChan := make(chan query.Result, len(entries)) - for _, entry := range entries { - resultChan <- query.Result{Entry: query.Entry{Value: entry}} + resultChan := make(chan query.Result, len(results)) + for _, result := range results { + resultChan <- result } close(resultChan) queryResults.EXPECT().Next().Return(resultChan).Maybe() diff --git a/db/collection.go b/db/collection.go index 4abfc9b4f6..2bd6d5afd0 100644 --- a/db/collection.go +++ b/db/collection.go @@ -56,6 +56,9 @@ type collection struct { schemaID string desc client.CollectionDescription + + isIndexCached bool + indexes []CollectionIndex } // @todo: Move the base Descriptions to an internal API within the db/ package. @@ -674,11 +677,13 @@ func (c *collection) SchemaID() string { // handle instead of a raw DB handle. func (c *collection) WithTxn(txn datastore.Txn) client.Collection { return &collection{ - db: c.db, - txn: immutable.Some(txn), - desc: c.desc, - colID: c.colID, - schemaID: c.schemaID, + db: c.db, + txn: immutable.Some(txn), + desc: c.desc, + colID: c.colID, + schemaID: c.schemaID, + isIndexCached: c.isIndexCached, + indexes: c.indexes, } } diff --git a/db/index.go b/db/index.go index f3cc8b5c74..4f72d71a0e 100644 --- a/db/index.go +++ b/db/index.go @@ -214,7 +214,11 @@ func (c *collection) dropAllIndexes(ctx context.Context) error { return nil } -func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { +func (c *collection) getIndexes(ctx context.Context) ([]CollectionIndex, error) { + if c.isIndexCached { + return c.indexes, nil + } + prefix := core.NewCollectionIndexKey(c.Name(), "") txn, err := c.getTxn(ctx, false) if err != nil { @@ -224,7 +228,7 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, Prefix: prefix.ToString(), }) if err != nil { - //return nil, NewErrFailedToCreateCollectionQuery(err) + return nil, NewErrFailedToCreateCollectionQuery(err) } defer func() { if err := q.Close(); err != nil { @@ -232,18 +236,34 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, } }() - indexDescriptions := make([]client.IndexDescription, 0) + indexes := make([]CollectionIndex, 0) for res := range q.Next() { if res.Error != nil { - //return nil, err + return nil, res.Error } var indexDesc client.IndexDescription err = json.Unmarshal(res.Value, &indexDesc) if err != nil { - //return nil, NewErrInvalidStoredIndex(err) + return nil, NewErrInvalidStoredIndex(err) } - indexDescriptions = append(indexDescriptions, indexDesc) + colIndex := NewCollectionIndex(c, indexDesc) + indexes = append(indexes, colIndex) + } + + c.indexes = indexes + c.isIndexCached = true + return indexes, nil +} + +func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { + indexes, err := c.getIndexes(ctx) + if err != nil { + return nil, err + } + indexDescriptions := make([]client.IndexDescription, 0, len(indexes)) + for _, index := range indexes { + indexDescriptions = append(indexDescriptions, index.Description()) } return indexDescriptions, nil diff --git a/db/index_test.go b/db/index_test.go index d607376912..8ead5068d2 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -20,11 +20,14 @@ import ( ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/datastore/mocks" + "github.com/sourcenetwork/defradb/errors" ) const ( @@ -550,6 +553,117 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) assert.Equal(t, productsIndexDesc, productIndexes[0]) } +func TestCollectionGetIndexes_ShouldReturnIndexes(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + indexes, err := f.users.GetIndexes(f.ctx) + assert.NoError(t, err) + + require.Equal(t, 1, len(indexes)) + assert.Equal(t, testUsersColIndexName, indexes[0].Name) +} + +func TestCollectionGetIndexes_IfCalledAgain_ShouldReturnCached(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + _, err := f.users.GetIndexes(f.ctx) + require.NoError(t, err) + + mockedTxn := mocks.NewTxnWithMultistore(f.t) + + indexes, err := f.users.WithTxn(mockedTxn).GetIndexes(f.ctx) + require.NoError(t, err) + + require.Equal(t, 1, len(indexes)) + assert.Equal(t, testUsersColIndexName, indexes[0].Name) +} + +func TestCollectionGetIndexes_ShouldCloseQueryIterator(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore = mocks.NewDSReaderWriter(f.t) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() + queryResults := mocks.NewQueryResultsWithValues(f.t) + queryResults.EXPECT().Close().Unset() + queryResults.EXPECT().Close().Return(nil) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything). + Return(queryResults, nil) + + _, err := f.users.WithTxn(mockedTxn).GetIndexes(f.ctx) + assert.NoError(t, err) +} + +func TestCollectionGetIndexes_IfSystemStoreFails_ShouldNotCache(t *testing.T) { + testErr := errors.New("test error") + + testCases := []struct { + Name string + ExpectedError error + GetMockSystemstore func(t *testing.T) *mocks.DSReaderWriter + }{ + { + Name: "Query fails", + ExpectedError: testErr, + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Query(mock.Anything, mock.Anything).Unset() + store.EXPECT().Query(mock.Anything, mock.Anything).Return(nil, testErr) + return store + }, + }, + { + Name: "Query iterator fails", + ExpectedError: testErr, + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}), nil) + return store + }, + }, + { + Name: "Query iterator returns invalid value", + ExpectedError: NewErrInvalidStoredIndex(nil), + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithValues(t, []byte("invalid")), nil) + return store + }, + }, + } + + for _, testCase := range testCases { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore = testCase.GetMockSystemstore(t) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() + + _, err := f.users.WithTxn(mockedTxn).GetIndexes(f.ctx) + require.ErrorIs(t, err, testCase.ExpectedError) + + indexes, err := f.users.GetIndexes(f.ctx) + require.NoError(t, err) + + require.Equal(t, 1, len(indexes)) + assert.Equal(t, testUsersColIndexName, indexes[0].Name) + } +} + func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index cae8335e9d..baaf44f110 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -192,8 +192,8 @@ func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { systemStoreOn.Get(mock.Anything, mock.Anything).Unset() colIndexOnNameKey := core.NewCollectionIndexKey(f.users.Description().Name, testUsersColIndexName) - systemStoreOn.Get(mock.Anything, colIndexOnNameKey.ToDS()).Return(indexOnNameDescData, nil) - systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + systemStoreOn.Get(mock.Anything, colIndexOnNameKey.ToDS()).Maybe().Return(indexOnNameDescData, nil) + systemStoreOn.Get(mock.Anything, mock.Anything).Maybe().Return([]byte{}, nil) f.txn = mockedTxn return mockedTxn From 5f6e340889cbae91a9f9f77b6d6bcbbcf81418ea Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 17 May 2023 15:08:00 +0200 Subject: [PATCH 041/120] Check if failed to create a new txn --- datastore/mocks/RootStore.go | 537 +++++++++++++++++++++++++++++++++++ db/index.go | 2 +- db/index_test.go | 24 ++ 3 files changed, 562 insertions(+), 1 deletion(-) create mode 100644 datastore/mocks/RootStore.go diff --git a/datastore/mocks/RootStore.go b/datastore/mocks/RootStore.go new file mode 100644 index 0000000000..730725e68a --- /dev/null +++ b/datastore/mocks/RootStore.go @@ -0,0 +1,537 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + datastore "github.com/ipfs/go-datastore" + + mock "github.com/stretchr/testify/mock" + + query "github.com/ipfs/go-datastore/query" +) + +// RootStore is an autogenerated mock type for the RootStore type +type RootStore struct { + mock.Mock +} + +type RootStore_Expecter struct { + mock *mock.Mock +} + +func (_m *RootStore) EXPECT() *RootStore_Expecter { + return &RootStore_Expecter{mock: &_m.Mock} +} + +// Batch provides a mock function with given fields: ctx +func (_m *RootStore) Batch(ctx context.Context) (datastore.Batch, error) { + ret := _m.Called(ctx) + + var r0 datastore.Batch + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (datastore.Batch, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) datastore.Batch); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.Batch) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RootStore_Batch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Batch' +type RootStore_Batch_Call struct { + *mock.Call +} + +// Batch is a helper method to define mock.On call +// - ctx context.Context +func (_e *RootStore_Expecter) Batch(ctx interface{}) *RootStore_Batch_Call { + return &RootStore_Batch_Call{Call: _e.mock.On("Batch", ctx)} +} + +func (_c *RootStore_Batch_Call) Run(run func(ctx context.Context)) *RootStore_Batch_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *RootStore_Batch_Call) Return(_a0 datastore.Batch, _a1 error) *RootStore_Batch_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *RootStore_Batch_Call) RunAndReturn(run func(context.Context) (datastore.Batch, error)) *RootStore_Batch_Call { + _c.Call.Return(run) + return _c +} + +// Close provides a mock function with given fields: +func (_m *RootStore) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// RootStore_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type RootStore_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +func (_e *RootStore_Expecter) Close() *RootStore_Close_Call { + return &RootStore_Close_Call{Call: _e.mock.On("Close")} +} + +func (_c *RootStore_Close_Call) Run(run func()) *RootStore_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *RootStore_Close_Call) Return(_a0 error) *RootStore_Close_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *RootStore_Close_Call) RunAndReturn(run func() error) *RootStore_Close_Call { + _c.Call.Return(run) + return _c +} + +// Delete provides a mock function with given fields: ctx, key +func (_m *RootStore) Delete(ctx context.Context, key datastore.Key) error { + ret := _m.Called(ctx, key) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) error); ok { + r0 = rf(ctx, key) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// RootStore_Delete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Delete' +type RootStore_Delete_Call struct { + *mock.Call +} + +// Delete is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *RootStore_Expecter) Delete(ctx interface{}, key interface{}) *RootStore_Delete_Call { + return &RootStore_Delete_Call{Call: _e.mock.On("Delete", ctx, key)} +} + +func (_c *RootStore_Delete_Call) Run(run func(ctx context.Context, key datastore.Key)) *RootStore_Delete_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *RootStore_Delete_Call) Return(_a0 error) *RootStore_Delete_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *RootStore_Delete_Call) RunAndReturn(run func(context.Context, datastore.Key) error) *RootStore_Delete_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function with given fields: ctx, key +func (_m *RootStore) Get(ctx context.Context, key datastore.Key) ([]byte, error) { + ret := _m.Called(ctx, key) + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) ([]byte, error)); ok { + return rf(ctx, key) + } + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) []byte); ok { + r0 = rf(ctx, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, datastore.Key) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RootStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type RootStore_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *RootStore_Expecter) Get(ctx interface{}, key interface{}) *RootStore_Get_Call { + return &RootStore_Get_Call{Call: _e.mock.On("Get", ctx, key)} +} + +func (_c *RootStore_Get_Call) Run(run func(ctx context.Context, key datastore.Key)) *RootStore_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *RootStore_Get_Call) Return(value []byte, err error) *RootStore_Get_Call { + _c.Call.Return(value, err) + return _c +} + +func (_c *RootStore_Get_Call) RunAndReturn(run func(context.Context, datastore.Key) ([]byte, error)) *RootStore_Get_Call { + _c.Call.Return(run) + return _c +} + +// GetSize provides a mock function with given fields: ctx, key +func (_m *RootStore) GetSize(ctx context.Context, key datastore.Key) (int, error) { + ret := _m.Called(ctx, key) + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) (int, error)); ok { + return rf(ctx, key) + } + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) int); ok { + r0 = rf(ctx, key) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(context.Context, datastore.Key) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RootStore_GetSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSize' +type RootStore_GetSize_Call struct { + *mock.Call +} + +// GetSize is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *RootStore_Expecter) GetSize(ctx interface{}, key interface{}) *RootStore_GetSize_Call { + return &RootStore_GetSize_Call{Call: _e.mock.On("GetSize", ctx, key)} +} + +func (_c *RootStore_GetSize_Call) Run(run func(ctx context.Context, key datastore.Key)) *RootStore_GetSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *RootStore_GetSize_Call) Return(size int, err error) *RootStore_GetSize_Call { + _c.Call.Return(size, err) + return _c +} + +func (_c *RootStore_GetSize_Call) RunAndReturn(run func(context.Context, datastore.Key) (int, error)) *RootStore_GetSize_Call { + _c.Call.Return(run) + return _c +} + +// Has provides a mock function with given fields: ctx, key +func (_m *RootStore) Has(ctx context.Context, key datastore.Key) (bool, error) { + ret := _m.Called(ctx, key) + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) (bool, error)); ok { + return rf(ctx, key) + } + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) bool); ok { + r0 = rf(ctx, key) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, datastore.Key) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RootStore_Has_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Has' +type RootStore_Has_Call struct { + *mock.Call +} + +// Has is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +func (_e *RootStore_Expecter) Has(ctx interface{}, key interface{}) *RootStore_Has_Call { + return &RootStore_Has_Call{Call: _e.mock.On("Has", ctx, key)} +} + +func (_c *RootStore_Has_Call) Run(run func(ctx context.Context, key datastore.Key)) *RootStore_Has_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *RootStore_Has_Call) Return(exists bool, err error) *RootStore_Has_Call { + _c.Call.Return(exists, err) + return _c +} + +func (_c *RootStore_Has_Call) RunAndReturn(run func(context.Context, datastore.Key) (bool, error)) *RootStore_Has_Call { + _c.Call.Return(run) + return _c +} + +// NewTransaction provides a mock function with given fields: ctx, readOnly +func (_m *RootStore) NewTransaction(ctx context.Context, readOnly bool) (datastore.Txn, error) { + ret := _m.Called(ctx, readOnly) + + var r0 datastore.Txn + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, bool) (datastore.Txn, error)); ok { + return rf(ctx, readOnly) + } + if rf, ok := ret.Get(0).(func(context.Context, bool) datastore.Txn); ok { + r0 = rf(ctx, readOnly) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.Txn) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, bool) error); ok { + r1 = rf(ctx, readOnly) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RootStore_NewTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewTransaction' +type RootStore_NewTransaction_Call struct { + *mock.Call +} + +// NewTransaction is a helper method to define mock.On call +// - ctx context.Context +// - readOnly bool +func (_e *RootStore_Expecter) NewTransaction(ctx interface{}, readOnly interface{}) *RootStore_NewTransaction_Call { + return &RootStore_NewTransaction_Call{Call: _e.mock.On("NewTransaction", ctx, readOnly)} +} + +func (_c *RootStore_NewTransaction_Call) Run(run func(ctx context.Context, readOnly bool)) *RootStore_NewTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(bool)) + }) + return _c +} + +func (_c *RootStore_NewTransaction_Call) Return(_a0 datastore.Txn, _a1 error) *RootStore_NewTransaction_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *RootStore_NewTransaction_Call) RunAndReturn(run func(context.Context, bool) (datastore.Txn, error)) *RootStore_NewTransaction_Call { + _c.Call.Return(run) + return _c +} + +// Put provides a mock function with given fields: ctx, key, value +func (_m *RootStore) Put(ctx context.Context, key datastore.Key, value []byte) error { + ret := _m.Called(ctx, key, value) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key, []byte) error); ok { + r0 = rf(ctx, key, value) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// RootStore_Put_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Put' +type RootStore_Put_Call struct { + *mock.Call +} + +// Put is a helper method to define mock.On call +// - ctx context.Context +// - key datastore.Key +// - value []byte +func (_e *RootStore_Expecter) Put(ctx interface{}, key interface{}, value interface{}) *RootStore_Put_Call { + return &RootStore_Put_Call{Call: _e.mock.On("Put", ctx, key, value)} +} + +func (_c *RootStore_Put_Call) Run(run func(ctx context.Context, key datastore.Key, value []byte)) *RootStore_Put_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key), args[2].([]byte)) + }) + return _c +} + +func (_c *RootStore_Put_Call) Return(_a0 error) *RootStore_Put_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *RootStore_Put_Call) RunAndReturn(run func(context.Context, datastore.Key, []byte) error) *RootStore_Put_Call { + _c.Call.Return(run) + return _c +} + +// Query provides a mock function with given fields: ctx, q +func (_m *RootStore) Query(ctx context.Context, q query.Query) (query.Results, error) { + ret := _m.Called(ctx, q) + + var r0 query.Results + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, query.Query) (query.Results, error)); ok { + return rf(ctx, q) + } + if rf, ok := ret.Get(0).(func(context.Context, query.Query) query.Results); ok { + r0 = rf(ctx, q) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(query.Results) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, query.Query) error); ok { + r1 = rf(ctx, q) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RootStore_Query_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Query' +type RootStore_Query_Call struct { + *mock.Call +} + +// Query is a helper method to define mock.On call +// - ctx context.Context +// - q query.Query +func (_e *RootStore_Expecter) Query(ctx interface{}, q interface{}) *RootStore_Query_Call { + return &RootStore_Query_Call{Call: _e.mock.On("Query", ctx, q)} +} + +func (_c *RootStore_Query_Call) Run(run func(ctx context.Context, q query.Query)) *RootStore_Query_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(query.Query)) + }) + return _c +} + +func (_c *RootStore_Query_Call) Return(_a0 query.Results, _a1 error) *RootStore_Query_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *RootStore_Query_Call) RunAndReturn(run func(context.Context, query.Query) (query.Results, error)) *RootStore_Query_Call { + _c.Call.Return(run) + return _c +} + +// Sync provides a mock function with given fields: ctx, prefix +func (_m *RootStore) Sync(ctx context.Context, prefix datastore.Key) error { + ret := _m.Called(ctx, prefix) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Key) error); ok { + r0 = rf(ctx, prefix) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// RootStore_Sync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Sync' +type RootStore_Sync_Call struct { + *mock.Call +} + +// Sync is a helper method to define mock.On call +// - ctx context.Context +// - prefix datastore.Key +func (_e *RootStore_Expecter) Sync(ctx interface{}, prefix interface{}) *RootStore_Sync_Call { + return &RootStore_Sync_Call{Call: _e.mock.On("Sync", ctx, prefix)} +} + +func (_c *RootStore_Sync_Call) Run(run func(ctx context.Context, prefix datastore.Key)) *RootStore_Sync_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Key)) + }) + return _c +} + +func (_c *RootStore_Sync_Call) Return(_a0 error) *RootStore_Sync_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *RootStore_Sync_Call) RunAndReturn(run func(context.Context, datastore.Key) error) *RootStore_Sync_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewRootStore interface { + mock.TestingT + Cleanup(func()) +} + +// NewRootStore creates a new instance of RootStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRootStore(t mockConstructorTestingTNewRootStore) *RootStore { + mock := &RootStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/db/index.go b/db/index.go index 4f72d71a0e..46574b47e4 100644 --- a/db/index.go +++ b/db/index.go @@ -222,7 +222,7 @@ func (c *collection) getIndexes(ctx context.Context) ([]CollectionIndex, error) prefix := core.NewCollectionIndexKey(c.Name(), "") txn, err := c.getTxn(ctx, false) if err != nil { - //return nil, err + return nil, err } q, err := txn.Systemstore().Query(ctx, query.Query{ Prefix: prefix.ToString(), diff --git a/db/index_test.go b/db/index_test.go index 8ead5068d2..5183a2ff48 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -664,6 +664,30 @@ func TestCollectionGetIndexes_IfSystemStoreFails_ShouldNotCache(t *testing.T) { } } +func TestCollectionGetIndexes_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + testErr := errors.New("test error") + + workingRootStore := f.db.rootstore + mockedRootStore := mocks.NewRootStore(t) + f.db.rootstore = mockedRootStore + mockedRootStore.EXPECT().NewTransaction(mock.Anything, mock.Anything).Return(nil, testErr) + + _, err := f.users.GetIndexes(f.ctx) + require.ErrorIs(t, err, testErr) + + f.db.rootstore = workingRootStore + + indexes, err := f.users.GetIndexes(f.ctx) + require.NoError(t, err) + + require.Equal(t, 1, len(indexes)) + assert.Equal(t, testUsersColIndexName, indexes[0].Name) +} + func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() From 5d3d681b463922ab76bd12878615a1d9360080d7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 17 May 2023 16:48:36 +0200 Subject: [PATCH 042/120] Use cached values instead of system storage --- datastore/mocks/utils.go | 9 +++++++++ db/collection.go | 21 ++++++--------------- db/index.go | 13 ++++++++----- db/indexed_docs_test.go | 23 +++++++++++------------ 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go index e12e5dc077..ded4bcb695 100644 --- a/datastore/mocks/utils.go +++ b/datastore/mocks/utils.go @@ -10,6 +10,7 @@ import ( type MultiStoreTxn struct { *Txn + t *testing.T MockRootstore *DSReaderWriter MockDatastore *DSReaderWriter MockHeadstore *DSReaderWriter @@ -60,6 +61,7 @@ func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { result := &MultiStoreTxn{ Txn: txn, + t: t, MockRootstore: prepareRootStore(t), MockDatastore: prepareDataStore(t), MockHeadstore: prepareHeadStore(t), @@ -76,6 +78,13 @@ func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { return result } +func (txn *MultiStoreTxn) ClearSystemStore() *MultiStoreTxn { + txn.MockSystemstore = NewDSReaderWriter(txn.t) + txn.EXPECT().Systemstore().Unset() + txn.EXPECT().Systemstore().Return(txn.MockSystemstore).Maybe() + return txn +} + func NewQueryResultsWithValues(t *testing.T, values ...[]byte) *Results { results := make([]query.Result, len(values)) for i, value := range values { diff --git a/db/collection.go b/db/collection.go index 2bd6d5afd0..5e32ad0a97 100644 --- a/db/collection.go +++ b/db/collection.go @@ -788,27 +788,18 @@ func (c *collection) create(ctx context.Context, txn datastore.Txn, doc *client. } func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { - indexes, err := c.db.getCollectionIndexes(ctx, txn, c.desc.Name) - err = err + indexes, err := c.getIndexes(ctx, txn) + if err != nil { + return err + } for _, index := range indexes { - indexedFieldName := index.Fields[0].Name + indexedFieldName := index.Description().Fields[0].Name fieldVal, err := doc.Get(indexedFieldName) if err != nil { return nil } - colIndexKey := core.NewCollectionIndexKey(c.desc.Name, index.Name) - indexData, err := txn.Systemstore().Get(ctx, colIndexKey.ToDS()) - if err != nil { - return NewErrFailedToReadStoredIndexDesc(err) - } - var indexDesc client.IndexDescription - err = json.Unmarshal(indexData, &indexDesc) - if err != nil { - return NewErrInvalidStoredIndex(err) - } - colIndex := NewCollectionIndex(c, indexDesc) docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) - err = colIndex.Save(ctx, txn, docDataStoreKey, fieldVal) + err = index.Save(ctx, txn, docDataStoreKey, fieldVal) if err != nil { return err } diff --git a/db/index.go b/db/index.go index 46574b47e4..4092033ec1 100644 --- a/db/index.go +++ b/db/index.go @@ -214,15 +214,18 @@ func (c *collection) dropAllIndexes(ctx context.Context) error { return nil } -func (c *collection) getIndexes(ctx context.Context) ([]CollectionIndex, error) { +func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]CollectionIndex, error) { if c.isIndexCached { return c.indexes, nil } prefix := core.NewCollectionIndexKey(c.Name(), "") - txn, err := c.getTxn(ctx, false) - if err != nil { - return nil, err + if txn == nil { + var err error + txn, err = c.getTxn(ctx, true) + if err != nil { + return nil, err + } } q, err := txn.Systemstore().Query(ctx, query.Query{ Prefix: prefix.ToString(), @@ -257,7 +260,7 @@ func (c *collection) getIndexes(ctx context.Context) ([]CollectionIndex, error) } func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { - indexes, err := c.getIndexes(ctx) + indexes, err := c.getIndexes(ctx, nil) if err != nil { return nil, err } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index baaf44f110..041e8be2f8 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -232,6 +232,7 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { require.ErrorIs(f.t, err, NewErrFailedToStoreIndexedField("name", nil)) } +// @todo: should store as nil value? func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -255,16 +256,14 @@ func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { func TestNonUnique_IfSystemStorageHasInvalidIndexDescription_Error(t *testing.T) { f := newIndexTestFixture(t) - indexDesc := f.createUserCollectionIndexOnName() + f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21) - mockTxn := f.mockTxn() + mockTxn := f.mockTxn().ClearSystemStore() systemStoreOn := mockTxn.MockSystemstore.EXPECT() - systemStoreOn.Get(mock.Anything, mock.Anything).Unset() - colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, indexDesc.Name) - systemStoreOn.Get(mock.Anything, colIndexKey.ToDS()).Return([]byte("invalid"), nil) - systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + systemStoreOn.Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithValues(t, []byte("invalid")), nil) err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) require.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) @@ -276,15 +275,15 @@ func TestNonUnique_IfSystemStorageFailsToReadIndexDesc_Error(t *testing.T) { doc := f.newUserDoc("John", 21) - mockTxn := f.mockTxn() + testErr := errors.New("test error") + + mockTxn := f.mockTxn().ClearSystemStore() systemStoreOn := mockTxn.MockSystemstore.EXPECT() - systemStoreOn.Get(mock.Anything, mock.Anything).Unset() - colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, testUsersColIndexName) - systemStoreOn.Get(mock.Anything, colIndexKey.ToDS()).Return([]byte{}, errors.New("error")) - systemStoreOn.Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + systemStoreOn.Query(mock.Anything, mock.Anything). + Return(nil, testErr) err := f.users.WithTxn(mockTxn).Create(f.ctx, doc) - require.ErrorIs(t, err, NewErrFailedToReadStoredIndexDesc(nil)) + require.ErrorIs(t, err, testErr) } func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { From af4338ad7e946eaf0df854250d64780fa464d75d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 17 May 2023 18:28:01 +0200 Subject: [PATCH 043/120] Make index check if doc is indexable --- db/collection.go | 8 +------- db/index.go | 15 ++++++++++----- db/index_test.go | 42 ++++++++++++++++++++--------------------- db/indexed_docs_test.go | 2 ++ 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/db/collection.go b/db/collection.go index 5e32ad0a97..64cf2f6913 100644 --- a/db/collection.go +++ b/db/collection.go @@ -793,13 +793,7 @@ func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *cl return err } for _, index := range indexes { - indexedFieldName := index.Description().Fields[0].Name - fieldVal, err := doc.Get(indexedFieldName) - if err != nil { - return nil - } - docDataStoreKey := c.getDSKeyFromDockey(doc.Key()) - err = index.Save(ctx, txn, docDataStoreKey, fieldVal) + err = index.Save(ctx, txn, doc) if err != nil { return err } diff --git a/db/index.go b/db/index.go index 4092033ec1..8b7bee6cf9 100644 --- a/db/index.go +++ b/db/index.go @@ -18,7 +18,7 @@ import ( ) type CollectionIndex interface { - Save(context.Context, datastore.Txn, core.DataStoreKey, any) error + Save(context.Context, datastore.Txn, *client.Document) error Name() string Description() client.IndexDescription } @@ -97,17 +97,22 @@ var _ CollectionIndex = (*collectionSimpleIndex)(nil) func (i *collectionSimpleIndex) Save( ctx context.Context, txn datastore.Txn, - key core.DataStoreKey, - val any, + doc *client.Document, ) error { - data, err := i.convertFunc(val) + indexedFieldName := i.desc.Fields[0].Name + fieldVal, err := doc.Get(indexedFieldName) + if err != nil { + return nil + } + + data, err := i.convertFunc(fieldVal) if err != nil { return NewErrCanNotIndexInvalidFieldValue(err) } indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) - indexDataStoreKey.FieldValues = []string{string(data), key.DocKey} + indexDataStoreKey.FieldValues = []string{string(data), doc.Key().String()} keyStr := indexDataStoreKey.ToDS() err = txn.Datastore().Put(ctx, keyStr, []byte{}) if err != nil { diff --git a/db/index_test.go b/db/index_test.go index 5183a2ff48..c640945e6d 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -553,6 +553,27 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) assert.Equal(t, productsIndexDesc, productIndexes[0]) } +func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + f.db.Close(f.ctx) + + _, err := f.getCollectionIndexes(usersColName) + assert.Error(t, err) +} + +func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) + assert.NoError(t, err) + + _, err = f.getCollectionIndexes(usersColName) + assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) +} + func TestCollectionGetIndexes_ShouldReturnIndexes(t *testing.T) { f := newIndexTestFixture(t) @@ -688,27 +709,6 @@ func TestCollectionGetIndexes_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { assert.Equal(t, testUsersColIndexName, indexes[0].Name) } -func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) - f.createUserCollectionIndexOnName() - - f.db.Close(f.ctx) - - _, err := f.getCollectionIndexes(usersColName) - assert.Error(t, err) -} - -func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) - - indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") - err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) - assert.NoError(t, err) - - _, err = f.getCollectionIndexes(usersColName) - assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) -} - func TestCollectionGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { f := newIndexTestFixture(t) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 041e8be2f8..e4a1353dd6 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -359,8 +359,10 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { testCase := []struct { Name string FieldKind client.FieldKind + // FieldVal is the value the index will receive for serialization FieldVal any ShouldFail bool + // Stored is the value that is stored as part of the index value key Stored string }{ {Name: "invalid int", FieldKind: client.FieldKind_INT, FieldVal: "invalid", ShouldFail: true}, From 89c39de9bccbb51cd6fd9bbb2b0548907826435c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 May 2023 09:56:39 +0200 Subject: [PATCH 044/120] Update cache on new index and on deleting an index --- db/index.go | 17 ++++++++++++++++- db/index_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/db/index.go b/db/index.go index 8b7bee6cf9..13eeb7e38e 100644 --- a/db/index.go +++ b/db/index.go @@ -176,6 +176,9 @@ func (c *collection) CreateIndex( if err != nil { return client.IndexDescription{}, err } + if c.isIndexCached { + c.indexes = append(c.indexes, index) + } return index.Description(), nil } @@ -186,7 +189,19 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { if err != nil { return err } - return txn.Systemstore().Delete(ctx, key.ToDS()) + err = txn.Systemstore().Delete(ctx, key.ToDS()) + if err != nil { + return err + } + if c.isIndexCached { + for i := range c.indexes { + if c.indexes[i].Name() == indexName { + c.indexes = append(c.indexes[:i], c.indexes[i+1:]...) + break + } + } + } + return nil } func (c *collection) dropAllIndexes(ctx context.Context) error { diff --git a/db/index_test.go b/db/index_test.go index c640945e6d..613aa2bcf2 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -725,6 +725,49 @@ func TestCollectionGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { require.ElementsMatch(t, []uint32{1, 2}, []uint32{indexes[0].ID, indexes[1].ID}) } +func TestCollectionGetIndexes_IfIndexIsCreated_ShouldUpdateCache(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + indexes, err := f.users.GetIndexes(f.ctx) + assert.NoError(t, err) + assert.Len(t, indexes, 1) + + _, err = f.users.CreateIndex(f.ctx, getUsersIndexDescOnAge()) + assert.NoError(t, err) + + indexes, err = f.users.GetIndexes(f.ctx) + assert.NoError(t, err) + assert.Len(t, indexes, 2) +} + +func TestCollectionGetIndexes_IfIndexIsDropped_ShouldUpdateCache(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + f.createUserCollectionIndexOnAge() + + indexes, err := f.users.GetIndexes(f.ctx) + assert.NoError(t, err) + assert.Len(t, indexes, 2) + + err = f.users.DropIndex(f.ctx, testUsersColIndexName) + assert.NoError(t, err) + + indexes, err = f.users.GetIndexes(f.ctx) + assert.NoError(t, err) + assert.Len(t, indexes, 1) + assert.Equal(t, indexes[0].Name, testUsersColIndexAge) + + err = f.users.DropIndex(f.ctx, testUsersColIndexAge) + assert.NoError(t, err) + + indexes, err = f.users.GetIndexes(f.ctx) + assert.NoError(t, err) + assert.Len(t, indexes, 0) +} + func TestDropIndex_ShouldDeleteIndex(t *testing.T) { f := newIndexTestFixture(t) desc := f.createUserCollectionIndexOnName() From 89a5faf1e3425d40d9a978fbc5320ec934ad9fc7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 May 2023 12:04:22 +0200 Subject: [PATCH 045/120] Check if sequence fetching failed --- db/index.go | 3 +++ db/index_test.go | 30 ++++++++++++++++++++++++------ db/indexed_docs_test.go | 37 ++++++++++++++++++++++++++++--------- 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/db/index.go b/db/index.go index 13eeb7e38e..a614ddc2cd 100644 --- a/db/index.go +++ b/db/index.go @@ -317,6 +317,9 @@ func (c *collection) createIndex( } colSeq, err := c.db.getSequence(ctx, txn, fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, c.ID())) + if err != nil { + return nil, err + } colID, err := colSeq.next(ctx, txn) desc.ID = uint32(colID) diff --git a/db/index_test.go b/db/index_test.go index 613aa2bcf2..2ce8cb3a77 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -45,15 +45,16 @@ const ( testUsersColIndexName = "user_name" testUsersColIndexAge = "user_age" + + userColVersionID = "bafkreiefzlx2xsfaxixs24hcqwwqpa3nuqbutkapasymk3d5v4fxa4rlhy" ) type indexTestFixture struct { - ctx context.Context - db *implicitTxnDB - txn datastore.Txn - users client.Collection - products client.Collection - t *testing.T + ctx context.Context + db *implicitTxnDB + txn datastore.Txn + users client.Collection + t *testing.T } func getUsersCollectionDesc() client.CollectionDescription { @@ -522,6 +523,23 @@ func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { assert.ErrorIs(t, err, NewErrInvalidStoredIndexKey(key.String())) } +func TestGetIndexes_IfFailsToReadNextSeqNumber_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn() + onSystemStore := mockedTxn.MockSystemstore.EXPECT() + f.resetSystemStoreStubs(onSystemStore) + + seqKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, f.users.ID())) + onSystemStore.Get(f.ctx, seqKey.ToDS()).Return(nil, testErr) + f.stubSystemStore(onSystemStore) + + _, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnName()) + assert.ErrorIs(t, err, testErr) +} + func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) { f := newIndexTestFixture(t) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index e4a1353dd6..368e918b41 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -174,29 +174,48 @@ func (f *indexTestFixture) getPrefixFromDataStore(prefix string) [][]byte { func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { mockedTxn := mocks.NewTxnWithMultistore(f.t) + systemStoreOn := mockedTxn.MockSystemstore.EXPECT() + f.resetSystemStoreStubs(systemStoreOn) + f.stubSystemStore(systemStoreOn) + + f.txn = mockedTxn + return mockedTxn +} + +func (*indexTestFixture) resetSystemStoreStubs(systemStoreOn *mocks.DSReaderWriter_Expecter) { + systemStoreOn.Query(mock.Anything, mock.Anything).Unset() + systemStoreOn.Get(mock.Anything, mock.Anything).Unset() +} + +func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_Expecter) { desc := getUsersIndexDescOnName() desc.ID = 1 indexOnNameDescData, err := json.Marshal(desc) require.NoError(f.t, err) - systemStoreOn := mockedTxn.MockSystemstore.EXPECT() - colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, "") matchPrefixFunc := func(q query.Query) bool { return q.Prefix == colIndexKey.ToDS().String() } - systemStoreOn.Query(mock.Anything, mock.Anything).Unset() systemStoreOn.Query(mock.Anything, mock.MatchedBy(matchPrefixFunc)).Maybe(). Return(mocks.NewQueryResultsWithValues(f.t, indexOnNameDescData), nil) systemStoreOn.Query(mock.Anything, mock.Anything).Maybe(). Return(mocks.NewQueryResultsWithValues(f.t), nil) - systemStoreOn.Get(mock.Anything, mock.Anything).Unset() + colKey := core.NewCollectionKey(f.users.Name()) + systemStoreOn.Get(mock.Anything, colKey.ToDS()).Maybe().Return([]byte(userColVersionID), nil) + + colVersionIDKey := core.NewCollectionSchemaVersionKey(userColVersionID) + colDesc := getUsersCollectionDesc() + colDesc.ID = 1 + colDescBytes, err := json.Marshal(colDesc) + require.NoError(f.t, err) + systemStoreOn.Get(mock.Anything, colVersionIDKey.ToDS()).Maybe().Return(colDescBytes, nil) + colIndexOnNameKey := core.NewCollectionIndexKey(f.users.Description().Name, testUsersColIndexName) systemStoreOn.Get(mock.Anything, colIndexOnNameKey.ToDS()).Maybe().Return(indexOnNameDescData, nil) systemStoreOn.Get(mock.Anything, mock.Anything).Maybe().Return([]byte{}, nil) - f.txn = mockedTxn - return mockedTxn + systemStoreOn.Has(mock.Anything, mock.Anything).Maybe().Return(false, nil) } func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { @@ -357,13 +376,13 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { nowStr := now.Format(time.RFC3339) testCase := []struct { - Name string - FieldKind client.FieldKind + Name string + FieldKind client.FieldKind // FieldVal is the value the index will receive for serialization FieldVal any ShouldFail bool // Stored is the value that is stored as part of the index value key - Stored string + Stored string }{ {Name: "invalid int", FieldKind: client.FieldKind_INT, FieldVal: "invalid", ShouldFail: true}, {Name: "invalid float", FieldKind: client.FieldKind_FLOAT, FieldVal: "invalid", ShouldFail: true}, From 45f72fa5dec02722d74f6922437151692490056b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 May 2023 20:39:27 +0200 Subject: [PATCH 046/120] Check if sequence incremented successfully --- db/index.go | 3 +++ db/index_test.go | 44 ++++++++++++++++++++++++++++++----------- db/indexed_docs_test.go | 10 ++++++++++ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/db/index.go b/db/index.go index a614ddc2cd..4aa4ea8b46 100644 --- a/db/index.go +++ b/db/index.go @@ -321,6 +321,9 @@ func (c *collection) createIndex( return nil, err } colID, err := colSeq.next(ctx, txn) + if err != nil { + return nil, err + } desc.ID = uint32(colID) buf, err := json.Marshal(desc) diff --git a/db/index_test.go b/db/index_test.go index 2ce8cb3a77..e0a2d3a228 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -45,7 +45,7 @@ const ( testUsersColIndexName = "user_name" testUsersColIndexAge = "user_age" - + userColVersionID = "bafkreiefzlx2xsfaxixs24hcqwwqpa3nuqbutkapasymk3d5v4fxa4rlhy" ) @@ -523,21 +523,41 @@ func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { assert.ErrorIs(t, err, NewErrInvalidStoredIndexKey(key.String())) } -func TestGetIndexes_IfFailsToReadNextSeqNumber_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) - +func TestGetIndexes_IfFailsToReadSeqNumber_ReturnError(t *testing.T) { testErr := errors.New("test error") - mockedTxn := f.mockTxn() - onSystemStore := mockedTxn.MockSystemstore.EXPECT() - f.resetSystemStoreStubs(onSystemStore) + testCases := []struct { + Name string + StubSystemStore func(*mocks.DSReaderWriter_Expecter, core.Key) + }{ + { + Name: "Read Sequence Number", + StubSystemStore: func(onSystemStore *mocks.DSReaderWriter_Expecter, seqKey core.Key) { + onSystemStore.Get(mock.Anything, seqKey.ToDS()).Return(nil, testErr) + }, + }, + { + Name: "Increment Sequence Number", + StubSystemStore: func(onSystemStore *mocks.DSReaderWriter_Expecter, seqKey core.Key) { + onSystemStore.Put(mock.Anything, seqKey.ToDS(), mock.Anything).Return(testErr) + }, + }, + } - seqKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, f.users.ID())) - onSystemStore.Get(f.ctx, seqKey.ToDS()).Return(nil, testErr) - f.stubSystemStore(onSystemStore) + for _, tc := range testCases { + f := newIndexTestFixture(t) - _, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnName()) - assert.ErrorIs(t, err, testErr) + mockedTxn := f.mockTxn() + onSystemStore := mockedTxn.MockSystemstore.EXPECT() + f.resetSystemStoreStubs(onSystemStore) + + seqKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, f.users.ID())) + tc.StubSystemStore(onSystemStore, seqKey) + f.stubSystemStore(onSystemStore) + + _, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnName()) + assert.ErrorIs(t, err, testErr) + } } func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 368e918b41..1194e62552 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -185,6 +185,7 @@ func (f *indexTestFixture) mockTxn() *mocks.MultiStoreTxn { func (*indexTestFixture) resetSystemStoreStubs(systemStoreOn *mocks.DSReaderWriter_Expecter) { systemStoreOn.Query(mock.Anything, mock.Anything).Unset() systemStoreOn.Get(mock.Anything, mock.Anything).Unset() + systemStoreOn.Put(mock.Anything, mock.Anything, mock.Anything).Unset() } func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_Expecter) { @@ -207,14 +208,23 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E colVersionIDKey := core.NewCollectionSchemaVersionKey(userColVersionID) colDesc := getUsersCollectionDesc() colDesc.ID = 1 + for i := range colDesc.Schema.Fields { + colDesc.Schema.Fields[i].ID = client.FieldID(i) + } colDescBytes, err := json.Marshal(colDesc) require.NoError(f.t, err) systemStoreOn.Get(mock.Anything, colVersionIDKey.ToDS()).Maybe().Return(colDescBytes, nil) colIndexOnNameKey := core.NewCollectionIndexKey(f.users.Description().Name, testUsersColIndexName) systemStoreOn.Get(mock.Anything, colIndexOnNameKey.ToDS()).Maybe().Return(indexOnNameDescData, nil) + + sequenceKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, f.users.ID())) + systemStoreOn.Get(mock.Anything, sequenceKey.ToDS()).Maybe().Return([]byte{0, 0, 0, 0, 0, 0, 0, 1}, nil) + systemStoreOn.Get(mock.Anything, mock.Anything).Maybe().Return([]byte{}, nil) + systemStoreOn.Put(mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil) + systemStoreOn.Has(mock.Anything, mock.Anything).Maybe().Return(false, nil) } From e96fe21a0666dcd84cf91402f7ffef8bf406c8fa Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 22 May 2023 10:16:44 +0200 Subject: [PATCH 047/120] Polish --- db/index_test.go | 60 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/db/index_test.go b/db/index_test.go index e0a2d3a228..dd60c88647 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -461,6 +461,36 @@ func TestCreateIndex_IfPropertyDoesntExist_ReturnError(t *testing.T) { assert.ErrorIs(t, err, NewErrNonExistingFieldForIndex(field)) } +func TestCreateIndex_WithMultipleCollectionsAndIndexes_AssignIncrementedIDPerCollection(t *testing.T) { + f := newIndexTestFixtureBare(t) + users := f.createCollection(getUsersCollectionDesc()) + products := f.createCollection(getProductsCollectionDesc()) + + makeIndex := func(fieldName string) client.IndexDescription { + return client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + {Name: fieldName, Direction: client.Ascending}, + }, + } + } + + createIndexAndAssert := func(col client.Collection, fieldName string, expectedID uint32) { + desc, err := f.createCollectionIndexFor(col.Name(), makeIndex(fieldName)) + require.NoError(t, err) + assert.Equal(t, expectedID, desc.ID) + seqKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, col.ID())) + storedSeqKey, err := f.txn.Systemstore().Get(f.ctx, seqKey.ToDS()) + assert.NoError(t, err) + storedSeqVal := binary.BigEndian.Uint64(storedSeqKey) + assert.Equal(t, expectedID, uint32(storedSeqVal)) + } + + createIndexAndAssert(users, usersNameFieldName, 1) + createIndexAndAssert(users, usersAgeFieldName, 2) + createIndexAndAssert(products, productsIDFieldName, 1) + createIndexAndAssert(products, productsCategoryFieldName, 2) +} + func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { f := newIndexTestFixture(t) @@ -868,33 +898,3 @@ func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { err := f.dropAllIndexes(usersColName) assert.Error(t, err) } - -func TestCreateIndex_WithMultipleCollectionsAndIndexes_AssignIncrementedIDPerCollection(t *testing.T) { - f := newIndexTestFixtureBare(t) - users := f.createCollection(getUsersCollectionDesc()) - products := f.createCollection(getProductsCollectionDesc()) - - makeIndex := func(fieldName string) client.IndexDescription { - return client.IndexDescription{ - Fields: []client.IndexedFieldDescription{ - {Name: fieldName, Direction: client.Ascending}, - }, - } - } - - createIndexAndAssert := func(col client.Collection, fieldName string, expectedID uint32) { - desc, err := f.createCollectionIndexFor(col.Name(), makeIndex(fieldName)) - require.NoError(t, err) - assert.Equal(t, expectedID, desc.ID) - seqKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, col.ID())) - storedSeqKey, err := f.txn.Systemstore().Get(f.ctx, seqKey.ToDS()) - assert.NoError(t, err) - storedSeqVal := binary.BigEndian.Uint64(storedSeqKey) - assert.Equal(t, expectedID, uint32(storedSeqVal)) - } - - createIndexAndAssert(users, usersNameFieldName, 1) - createIndexAndAssert(users, usersAgeFieldName, 2) - createIndexAndAssert(products, productsIDFieldName, 1) - createIndexAndAssert(products, productsCategoryFieldName, 2) -} From 0e6ad3259246d808860cc083bf3a7020a4ad154e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 22 May 2023 12:15:43 +0200 Subject: [PATCH 048/120] Use "v" prefix for field values --- core/key_test.go | 2 +- db/index.go | 7 ++++++- db/indexed_docs_test.go | 8 +++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/key_test.go b/core/key_test.go index 1ca1b836db..d0b0fb104f 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -114,7 +114,7 @@ func TestNewIndexKey_IfEmptyParam_ReturnPrefix(t *testing.T) { assert.Equal(t, "/collection/index", key.ToString()) } -func TestNewIndexKey_ParamsAreGiven_ReturnFullKey(t *testing.T) { +func TestNewIndexKey_IfParamsAreGiven_ReturnFullKey(t *testing.T) { key := NewCollectionIndexKey("col", "idx") assert.Equal(t, "/collection/index/col/idx", key.ToString()) } diff --git a/db/index.go b/db/index.go index 4aa4ea8b46..f15de5e9ed 100644 --- a/db/index.go +++ b/db/index.go @@ -17,6 +17,8 @@ import ( "github.com/sourcenetwork/defradb/errors" ) +const indexFieldValuePrefix = "v" + type CollectionIndex interface { Save(context.Context, datastore.Txn, *client.Document) error Name() string @@ -112,7 +114,10 @@ func (i *collectionSimpleIndex) Save( indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) - indexDataStoreKey.FieldValues = []string{string(data), doc.Key().String()} + indexDataStoreKey.FieldValues = []string{ + indexFieldValuePrefix + string(data), + indexFieldValuePrefix + doc.Key().String(), + } keyStr := indexDataStoreKey.ToDS() err = txn.Datastore().Put(ctx, keyStr, []byte{}) if err != nil { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 1194e62552..57a66f8d94 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -17,6 +17,8 @@ import ( "github.com/stretchr/testify/require" ) +const testValuePrefix = "v" + type userDoc struct { Name string `json:"name"` Age int `json:"age"` @@ -146,12 +148,12 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { if len(b.values) == 0 { fieldVal, err := b.doc.Get(b.fieldName) require.NoError(b.f.t, err) - fieldStrVal = fmt.Sprintf("%v", fieldVal) + fieldStrVal = fmt.Sprintf("%s%v", testValuePrefix, fieldVal) } else { fieldStrVal = b.values[0] } - key.FieldValues = []string{fieldStrVal, b.doc.Key().String()} + key.FieldValues = []string{fieldStrVal, testValuePrefix + b.doc.Key().String()} } else if len(b.values) > 0 { key.FieldValues = b.values } @@ -454,7 +456,7 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { keyBuilder := newIndexKeyBuilder(f).Col(collection.Name()).Field("field").Doc(doc) if tc.Stored != "" { - keyBuilder.Values(tc.Stored) + keyBuilder.Values(testValuePrefix + tc.Stored) } key := keyBuilder.Build() From e7dfc4365a22d817c0ce0ac1bee597b9d4c2e845 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 22 May 2023 13:35:57 +0200 Subject: [PATCH 049/120] Store nil index field with a special value --- db/index.go | 28 +++++++++++++++++++--------- db/indexed_docs_test.go | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/db/index.go b/db/index.go index f15de5e9ed..c48fbf8824 100644 --- a/db/index.go +++ b/db/index.go @@ -17,7 +17,10 @@ import ( "github.com/sourcenetwork/defradb/errors" ) -const indexFieldValuePrefix = "v" +const ( + indexFieldValuePrefix = "v" + indexFieldNilValue = "n" +) type CollectionIndex interface { Save(context.Context, datastore.Txn, *client.Document) error @@ -103,21 +106,28 @@ func (i *collectionSimpleIndex) Save( ) error { indexedFieldName := i.desc.Fields[0].Name fieldVal, err := doc.Get(indexedFieldName) + isNil := false if err != nil { - return nil + isNil = errors.Is(err, client.ErrFieldNotExist) + if !isNil { + return nil + } } - data, err := i.convertFunc(fieldVal) - if err != nil { - return NewErrCanNotIndexInvalidFieldValue(err) + storeValue := "" + if isNil { + storeValue = indexFieldNilValue + } else { + data, err := i.convertFunc(fieldVal) + if err != nil { + return NewErrCanNotIndexInvalidFieldValue(err) + } + storeValue = indexFieldValuePrefix + string(data) } indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) - indexDataStoreKey.FieldValues = []string{ - indexFieldValuePrefix + string(data), - indexFieldValuePrefix + doc.Key().String(), - } + indexDataStoreKey.FieldValues = []string{storeValue, indexFieldValuePrefix + doc.Key().String()} keyStr := indexDataStoreKey.ToDS() err = txn.Datastore().Put(ctx, keyStr, []byte{}) if err != nil { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 57a66f8d94..53a1f71e2d 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -18,6 +18,7 @@ import ( ) const testValuePrefix = "v" +const testNilValue = "n" type userDoc struct { Name string `json:"name"` @@ -467,3 +468,25 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { } } } + +func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + docJSON, err := json.Marshal(struct { + Age int `json:"age"` + }{Age: 44}) + require.NoError(f.t, err) + + doc, err := client.NewDocFromJSON(docJSON) + require.NoError(f.t, err) + + f.saveToUsers(doc) + + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). + Values(testNilValue).Build() + + data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} From d165c5ddf2be886b8221a57c49d4f4af044e4b3c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 22 May 2023 13:39:28 +0200 Subject: [PATCH 050/120] Add test for empty string --- db/indexed_docs_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 53a1f71e2d..966aaa0269 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -407,6 +407,7 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { {Name: "valid bool true", FieldKind: client.FieldKind_BOOL, FieldVal: true, Stored: "1"}, {Name: "valid bool false", FieldKind: client.FieldKind_BOOL, FieldVal: false, Stored: "0"}, {Name: "valid datetime string", FieldKind: client.FieldKind_DATETIME, FieldVal: nowStr, Stored: nowStr}, + {Name: "valid empty string", FieldKind: client.FieldKind_STRING, FieldVal: "", Stored: ""}, } for i, tc := range testCase { From c27e6db335e9b72f52034eefefd5320461ff475d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 23 May 2023 12:25:39 +0200 Subject: [PATCH 051/120] Remove indexed fields upon dropping index --- db/errors.go | 8 ++- db/index.go | 49 +++++++++++++++-- db/index_test.go | 29 +++++++++- db/indexed_docs_test.go | 119 ++++++++++++++++++++++++++++++++++++++-- 4 files changed, 190 insertions(+), 15 deletions(-) diff --git a/db/errors.go b/db/errors.go index c78c7e204e..d809c54c87 100644 --- a/db/errors.go +++ b/db/errors.go @@ -50,6 +50,7 @@ const ( errFailedToStoreIndexedField string = "failed to store indexed field" errFailedToReadStoredIndexDesc string = "failed to read stored index description" errCanNotIndexInvalidFieldValue string = "can not index invalid field value" + errCanNotDeleteIndexedField string = "can not delete indexed field" ) var ( @@ -150,12 +151,17 @@ func NewErrFailedToReadStoredIndexDesc(inner error) error { return errors.Wrap(errFailedToReadStoredIndexDesc, inner) } -// NewErrCanNotIndexInvalidFieldValue returns a new error indicating that the field value is invalid +// NewErrCanNotIndexInvalidFieldValue returns a new error indicating that the field value is invalid // and cannot be indexed. func NewErrCanNotIndexInvalidFieldValue(inner error) error { return errors.Wrap(errCanNotIndexInvalidFieldValue, inner) } +// NewCanNotDeleteIndexedField returns a new error a failed attempt to delete an indexed field +func NewCanNotDeleteIndexedField(inner error) error { + return errors.Wrap(errCanNotDeleteIndexedField, inner) +} + // NewErrNonZeroIndexIDProvided returns a new error indicating that a non-zero index ID was provided. func NewErrNonZeroIndexIDProvided(indexID uint32) error { return errors.New(errNonZeroIndexIDProvided, errors.NewKV("ID", indexID)) diff --git a/db/index.go b/db/index.go index c48fbf8824..a29f6b78be 100644 --- a/db/index.go +++ b/db/index.go @@ -24,6 +24,7 @@ const ( type CollectionIndex interface { Save(context.Context, datastore.Txn, *client.Document) error + RemoveAll(context.Context, datastore.Txn) error Name() string Description() client.IndexDescription } @@ -136,6 +137,35 @@ func (i *collectionSimpleIndex) Save( return nil } +func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn) error { + prefixKey := core.IndexDataStoreKey{} + prefixKey.CollectionID = strconv.Itoa(int(i.collection.ID())) + prefixKey.IndexID = strconv.Itoa(int(i.desc.ID)) + q, err := txn.Datastore().Query(ctx, query.Query{ + Prefix: prefixKey.ToString(), + }) + if err != nil { + return err + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + for res := range q.Next() { + if res.Error != nil { + return res.Error + } + err = txn.Datastore().Delete(ctx, ds.NewKey(res.Key)) + if err != nil { + return NewCanNotDeleteIndexedField(err) + } + } + + return nil +} + func (i *collectionSimpleIndex) Name() string { return i.desc.Name } @@ -204,18 +234,25 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { if err != nil { return err } - err = txn.Systemstore().Delete(ctx, key.ToDS()) + _, err = c.getIndexes(ctx, txn) if err != nil { return err } - if c.isIndexCached { - for i := range c.indexes { - if c.indexes[i].Name() == indexName { - c.indexes = append(c.indexes[:i], c.indexes[i+1:]...) - break + for i := range c.indexes { + if c.indexes[i].Name() == indexName { + err = c.indexes[i].RemoveAll(ctx, txn) + if err != nil { + return err } + c.indexes = append(c.indexes[:i], c.indexes[i+1:]...) + break } } + err = txn.Systemstore().Delete(ctx, key.ToDS()) + if err != nil { + return err + } + return nil } diff --git a/db/index_test.go b/db/index_test.go index dd60c88647..a9ac2c6f15 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -43,8 +43,9 @@ const ( productsCategoryFieldName = "category" productsAvailableFieldName = "available" - testUsersColIndexName = "user_name" - testUsersColIndexAge = "user_age" + testUsersColIndexName = "user_name" + testUsersColIndexAge = "user_age" + testUsersColIndexWeight = "user_weight" userColVersionID = "bafkreiefzlx2xsfaxixs24hcqwwqpa3nuqbutkapasymk3d5v4fxa4rlhy" ) @@ -165,6 +166,15 @@ func getUsersIndexDescOnAge() client.IndexDescription { } } +func getUsersIndexDescOnWeight() client.IndexDescription { + return client.IndexDescription{ + Name: testUsersColIndexWeight, + Fields: []client.IndexedFieldDescription{ + {Name: usersWeightFieldName, Direction: client.Ascending}, + }, + } +} + func getProductsIndexDescOnCategory() client.IndexDescription { return client.IndexDescription{ Name: testUsersColIndexAge, @@ -865,6 +875,21 @@ func TestDropIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { assert.ErrorIs(t, err, NewErrCollectionDoesntExist(usersColName)) } +func TestDropIndex_IfFailsToQuerySystemStorage_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + desc := f.createUserCollectionIndexOnName() + + testErr := errors.New("test error") + + mockTxn := f.mockTxn().ClearSystemStore() + systemStoreOn := mockTxn.MockSystemstore.EXPECT() + systemStoreOn.Query(mock.Anything, mock.Anything).Return(nil, testErr) + f.stubSystemStore(systemStoreOn) + + err := f.dropIndex(usersColName, desc.Name) + require.ErrorIs(t, err, testErr) +} + func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { f := newIndexTestFixture(t) _, err := f.createCollectionIndexFor(usersColName, client.IndexDescription{ diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 966aaa0269..1bd989c57c 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -33,8 +33,8 @@ type productDoc struct { Available bool `json:"available"` } -func (f *indexTestFixture) saveToUsers(doc *client.Document) { - err := f.users.Create(f.ctx, doc) +func (f *indexTestFixture) saveDocToCollection(doc *client.Document, col client.Collection) { + err := col.Create(f.ctx, doc) require.NoError(f.t, err) f.txn, err = f.db.NewTxn(f.ctx, false) require.NoError(f.t, err) @@ -229,6 +229,8 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E systemStoreOn.Put(mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil) systemStoreOn.Has(mock.Anything, mock.Anything).Maybe().Return(false, nil) + + systemStoreOn.Delete(mock.Anything, mock.Anything).Maybe().Return(nil) } func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { @@ -236,7 +238,7 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21) - f.saveToUsers(doc) + f.saveDocToCollection(doc, f.users) //f.commitTxn() key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() @@ -323,7 +325,7 @@ func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { f.createUserCollectionIndexOnAge() doc := f.newUserDoc("John", 21) - f.saveToUsers(doc) + f.saveDocToCollection(doc, f.users) key := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() @@ -369,7 +371,7 @@ func TestNonUnique_IfMultipleIndexes_StoreIndexWithIndexID(t *testing.T) { f.createUserCollectionIndexOnAge() doc := f.newUserDoc("John", 21) - f.saveToUsers(doc) + f.saveDocToCollection(doc, f.users) nameKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() ageKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() @@ -482,7 +484,7 @@ func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { doc, err := client.NewDocFromJSON(docJSON) require.NoError(f.t, err) - f.saveToUsers(doc) + f.saveDocToCollection(doc, f.users) key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). Values(testNilValue).Build() @@ -491,3 +493,108 @@ func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { require.NoError(t, err) assert.Len(t, data, 0) } + +func TestNonUnique_IfIndexIsDropped_ShouldDeleteStoredIndexedFields(t *testing.T) { + f := newIndexTestFixtureBare(t) + users := f.createCollection(getUsersCollectionDesc()) + _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) + require.NoError(f.t, err) + _, err = f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnAge()) + require.NoError(f.t, err) + _, err = f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnWeight()) + require.NoError(f.t, err) + f.commitTxn() + + f.saveDocToCollection(f.newUserDoc("John", 21), users) + f.saveDocToCollection(f.newUserDoc("Islam", 23), users) + + products := f.createCollection(getProductsCollectionDesc()) + _, err = f.createCollectionIndexFor(products.Name(), getProductsIndexDescOnCategory()) + require.NoError(f.t, err) + f.commitTxn() + + f.saveDocToCollection(f.newProdDoc(1, 55, "games"), products) + + userNameKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Build() + userAgeKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Build() + userWeightKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersWeightFieldName).Build() + prodCatKey := newIndexKeyBuilder(f).Col(productsColName).Field(productsCategoryFieldName).Build() + + err = f.dropIndex(usersColName, testUsersColIndexAge) + require.NoError(f.t, err) + + assert.Len(t, f.getPrefixFromDataStore(userNameKey.ToString()), 2) + assert.Len(t, f.getPrefixFromDataStore(userAgeKey.ToString()), 0) + assert.Len(t, f.getPrefixFromDataStore(userWeightKey.ToString()), 2) + assert.Len(t, f.getPrefixFromDataStore(prodCatKey.ToString()), 1) +} + +func TestNonUnique_IfUponDroppingIndexFailsToQueryDataStorage_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn() + mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) + mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything).Unset() + mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything).Return(nil, testErr) + mockedTxn.EXPECT().Datastore().Unset() + mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) + + err := f.dropIndex(usersColName, testUsersColIndexName) + require.ErrorIs(t, err, testErr) +} + +func TestNonUnique_IfUponDroppingIndexQueryIteratorFails_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn() + mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) + mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}), nil) + mockedTxn.EXPECT().Datastore().Unset() + mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) + + err := f.dropIndex(usersColName, testUsersColIndexName) + require.ErrorIs(t, err, testErr) +} + +func TestNonUnique_IfUponDroppingIndex_ShouldCloseQueryIterator(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + mockedTxn := f.mockTxn() + + mockedTxn.MockDatastore = mocks.NewDSReaderWriter(f.t) + mockedTxn.EXPECT().Datastore().Unset() + mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore).Maybe() + queryResults := mocks.NewQueryResultsWithValues(f.t) + queryResults.EXPECT().Close().Unset() + queryResults.EXPECT().Close().Return(nil) + mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything). + Return(queryResults, nil) + + err := f.dropIndex(usersColName, testUsersColIndexName) + assert.NoError(t, err) +} + +func TestNonUnique_IfUponDroppingIndexDatastoreFailsToDelete_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + mockedTxn := f.mockTxn() + mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) + mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithValues(t, []byte{}), nil) + mockedTxn.MockDatastore.EXPECT().Delete(mock.Anything, mock.Anything). + Return(errors.New("error")) + mockedTxn.EXPECT().Datastore().Unset() + mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) + + err := f.dropIndex(usersColName, testUsersColIndexName) + require.ErrorIs(t, err, NewCanNotDeleteIndexedField(nil)) +} From 99b999ceed325cb81eb33a920367d9cc9e607bfa Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 23 May 2023 13:25:21 +0200 Subject: [PATCH 052/120] Rename tests --- db/indexed_docs_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 1bd989c57c..b9cbe03ad2 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -494,7 +494,7 @@ func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { assert.Len(t, data, 0) } -func TestNonUnique_IfIndexIsDropped_ShouldDeleteStoredIndexedFields(t *testing.T) { +func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f := newIndexTestFixtureBare(t) users := f.createCollection(getUsersCollectionDesc()) _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) @@ -529,7 +529,7 @@ func TestNonUnique_IfIndexIsDropped_ShouldDeleteStoredIndexedFields(t *testing.T assert.Len(t, f.getPrefixFromDataStore(prodCatKey.ToString()), 1) } -func TestNonUnique_IfUponDroppingIndexFailsToQueryDataStorage_ReturnError(t *testing.T) { +func TestNonUniqueDrop_IfFailsToQueryDataStorage_ReturnError(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -546,7 +546,7 @@ func TestNonUnique_IfUponDroppingIndexFailsToQueryDataStorage_ReturnError(t *tes require.ErrorIs(t, err, testErr) } -func TestNonUnique_IfUponDroppingIndexQueryIteratorFails_ReturnError(t *testing.T) { +func TestNonUniqueDrop_IfQueryIteratorFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -563,7 +563,7 @@ func TestNonUnique_IfUponDroppingIndexQueryIteratorFails_ReturnError(t *testing. require.ErrorIs(t, err, testErr) } -func TestNonUnique_IfUponDroppingIndex_ShouldCloseQueryIterator(t *testing.T) { +func TestNonUniqueDrop_ShouldCloseQueryIterator(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -582,7 +582,7 @@ func TestNonUnique_IfUponDroppingIndex_ShouldCloseQueryIterator(t *testing.T) { assert.NoError(t, err) } -func TestNonUnique_IfUponDroppingIndexDatastoreFailsToDelete_ReturnError(t *testing.T) { +func TestNonUniqueDrop_IfDatastoreFailsToDelete_ReturnError(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() From 9446ca00f51e5852ec60e980d40f1a2642825ab9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 23 May 2023 16:50:52 +0200 Subject: [PATCH 053/120] Extract prefix deserialization --- db/index.go | 55 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/db/index.go b/db/index.go index a29f6b78be..8057f2e31a 100644 --- a/db/index.go +++ b/db/index.go @@ -286,22 +286,8 @@ func (c *collection) dropAllIndexes(ctx context.Context) error { return nil } -func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]CollectionIndex, error) { - if c.isIndexCached { - return c.indexes, nil - } - - prefix := core.NewCollectionIndexKey(c.Name(), "") - if txn == nil { - var err error - txn, err = c.getTxn(ctx, true) - if err != nil { - return nil, err - } - } - q, err := txn.Systemstore().Query(ctx, query.Query{ - Prefix: prefix.ToString(), - }) +func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Read) ([]T, error) { + q, err := storage.Query(ctx, query.Query{Prefix: prefix}) if err != nil { return nil, NewErrFailedToCreateCollectionQuery(err) } @@ -311,24 +297,47 @@ func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]Colle } }() - indexes := make([]CollectionIndex, 0) + elements := make([]T, 0) for res := range q.Next() { if res.Error != nil { return nil, res.Error } - var indexDesc client.IndexDescription - err = json.Unmarshal(res.Value, &indexDesc) + var element T + err = json.Unmarshal(res.Value, &element) if err != nil { return nil, NewErrInvalidStoredIndex(err) } - colIndex := NewCollectionIndex(c, indexDesc) - indexes = append(indexes, colIndex) + elements = append(elements, element) + } + return elements, nil +} + +func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]CollectionIndex, error) { + if c.isIndexCached { + return c.indexes, nil + } + + prefix := core.NewCollectionIndexKey(c.Name(), "") + if txn == nil { + var err error + txn, err = c.getTxn(ctx, true) + if err != nil { + return nil, err + } + } + indexes, err := deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) + if err != nil { + return nil, err + } + colIndexes := make([]CollectionIndex, 0, len(indexes)) + for _, index := range indexes { + colIndexes = append(colIndexes, NewCollectionIndex(c, index)) } - c.indexes = indexes + c.indexes = colIndexes c.isIndexCached = true - return indexes, nil + return colIndexes, nil } func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { From 04d4c90ba21dd8364387b8b628fb89dc3e079d9a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 26 May 2023 13:40:32 +0200 Subject: [PATCH 054/120] Update index in doc update --- db/collection.go | 31 +++++++++++++++++++++++++++ db/index.go | 41 +++++++++++++++++++++++++++--------- db/indexed_docs_test.go | 46 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 10 deletions(-) diff --git a/db/collection.go b/db/collection.go index 64cf2f6913..e1dd14cce4 100644 --- a/db/collection.go +++ b/db/collection.go @@ -30,6 +30,7 @@ import ( "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/db/base" + "github.com/sourcenetwork/defradb/db/fetcher" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/logging" @@ -873,12 +874,42 @@ func (c *collection) Save(ctx context.Context, doc *client.Document) error { return c.commitImplicitTxn(ctx, txn) } +func (c *collection) updateIndex( + ctx context.Context, + txn datastore.Txn, + doc *client.Document, +) error { + f := new(fetcher.DocumentFetcher) + fields := make([]*client.FieldDescription, len(c.desc.Schema.Fields)) + for i, field := range c.desc.Schema.Fields { + fields[i] = &field + } + err := f.Init(&c.desc, fields, false, false) + err = err + + docKey := base.MakeDocKey(c.Description(), doc.Key().String()) + err = f.Start(ctx, txn, core.NewSpans(core.NewSpan(docKey, docKey.PrefixEnd()))) + oldDoc, err := f.FetchNextDecoded(ctx) + _, err = c.getIndexes(ctx, txn) + for _, index := range c.indexes { + err = index.Update(ctx, txn, oldDoc, doc) + } + err = f.Close() + return nil +} + func (c *collection) save( ctx context.Context, txn datastore.Txn, doc *client.Document, isCreate bool, ) (cid.Cid, error) { + if !isCreate { + err := c.updateIndex(ctx, txn, doc) + if err != nil { + return cid.Undef, err + } + } // NOTE: We delay the final Clean() call until we know // the commit on the transaction is successful. If we didn't // wait, and just did it here, then *if* the commit fails down diff --git a/db/index.go b/db/index.go index 8057f2e31a..d1e150250d 100644 --- a/db/index.go +++ b/db/index.go @@ -24,6 +24,7 @@ const ( type CollectionIndex interface { Save(context.Context, datastore.Txn, *client.Document) error + Update(context.Context, datastore.Txn, *client.Document, *client.Document) error RemoveAll(context.Context, datastore.Txn) error Name() string Description() client.IndexDescription @@ -100,18 +101,14 @@ type collectionSimpleIndex struct { var _ CollectionIndex = (*collectionSimpleIndex)(nil) -func (i *collectionSimpleIndex) Save( - ctx context.Context, - txn datastore.Txn, - doc *client.Document, -) error { +func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataStoreKey, error) { indexedFieldName := i.desc.Fields[0].Name fieldVal, err := doc.Get(indexedFieldName) isNil := false if err != nil { isNil = errors.Is(err, client.ErrFieldNotExist) if !isNil { - return nil + return core.IndexDataStoreKey{}, nil // @todo: test } } @@ -121,7 +118,7 @@ func (i *collectionSimpleIndex) Save( } else { data, err := i.convertFunc(fieldVal) if err != nil { - return NewErrCanNotIndexInvalidFieldValue(err) + return core.IndexDataStoreKey{}, NewErrCanNotIndexInvalidFieldValue(err) } storeValue = indexFieldValuePrefix + string(data) } @@ -129,11 +126,35 @@ func (i *collectionSimpleIndex) Save( indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) indexDataStoreKey.FieldValues = []string{storeValue, indexFieldValuePrefix + doc.Key().String()} - keyStr := indexDataStoreKey.ToDS() - err = txn.Datastore().Put(ctx, keyStr, []byte{}) + return indexDataStoreKey, nil +} + +func (i *collectionSimpleIndex) Save( + ctx context.Context, + txn datastore.Txn, + doc *client.Document, +) error { + key, err := i.getDocKey(doc) if err != nil { - return NewErrFailedToStoreIndexedField(indexDataStoreKey.IndexID, err) + return err } + err = txn.Datastore().Put(ctx, key.ToDS(), []byte{}) + if err != nil { + return NewErrFailedToStoreIndexedField(key.IndexID, err) + } + return nil +} + +func (i *collectionSimpleIndex) Update( + ctx context.Context, + txn datastore.Txn, + oldDoc *client.Document, + newDoc *client.Document, +) error { + key, err := i.getDocKey(oldDoc) + err = err + err = txn.Datastore().Delete(ctx, key.ToDS()) + i.Save(ctx, txn, newDoc) return nil } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index b9cbe03ad2..29f188ca48 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -598,3 +598,49 @@ func TestNonUniqueDrop_IfDatastoreFailsToDelete_ReturnError(t *testing.T) { err := f.dropIndex(usersColName, testUsersColIndexName) require.ErrorIs(t, err, NewCanNotDeleteIndexedField(nil)) } + +func TestNonUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + cases := []struct { + Name string + NewValue string + Exec func(doc *client.Document) error + }{ + { + Name: "update", + NewValue: "Islam", + Exec: func(doc *client.Document) error { + return f.users.Update(f.ctx, doc) + }, + }, + { + Name: "save", + NewValue: "Andy", + Exec: func(doc *client.Document) error { + return f.users.Save(f.ctx, doc) + }, + }, + } + + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + + for _, tc := range cases { + oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + + err := doc.Set(usersNameFieldName, tc.NewValue) + require.NoError(t, err) + err = tc.Exec(doc) + require.NoError(t, err) + f.commitTxn() + + newKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + + _, err = f.txn.Datastore().Get(f.ctx, oldKey.ToDS()) + require.Error(t, err) + _, err = f.txn.Datastore().Get(f.ctx, newKey.ToDS()) + require.NoError(t, err) + } +} From d995afabf1563344dc4f2dd53f117124de5c31c9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 26 May 2023 13:53:49 +0200 Subject: [PATCH 055/120] Move index related code to another file --- db/collection.go | 144 ----------------------------------- db/collection_index.go | 167 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 144 deletions(-) create mode 100644 db/collection_index.go diff --git a/db/collection.go b/db/collection.go index e1dd14cce4..3fcab67986 100644 --- a/db/collection.go +++ b/db/collection.go @@ -30,7 +30,6 @@ import ( "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/db/base" - "github.com/sourcenetwork/defradb/db/fetcher" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/logging" @@ -193,111 +192,6 @@ func (db *db) createCollection( return col, nil } -// createCollectionIndex creates a new collection index and saves it to the database in its system store. -func (db *db) createCollectionIndex( - ctx context.Context, - txn datastore.Txn, - collectionName string, - desc client.IndexDescription, -) (client.IndexDescription, error) { - col, err := db.getCollectionByName(ctx, txn, collectionName) - if err != nil { - return client.IndexDescription{}, NewErrCollectionDoesntExist(collectionName) - } - col = col.WithTxn(txn) - return col.CreateIndex(ctx, desc) -} - -func (db *db) dropCollectionIndex( - ctx context.Context, - txn datastore.Txn, - collectionName, indexName string, -) error { - col, err := db.getCollectionByName(ctx, txn, collectionName) - if err != nil { - return NewErrCollectionDoesntExist(collectionName) - } - col = col.WithTxn(txn) - return col.DropIndex(ctx, indexName) -} - -// getAllCollectionIndexes returns all the indexes in the database. -func (db *db) getAllCollectionIndexes( - ctx context.Context, - txn datastore.Txn, -) ([]client.CollectionIndexDescription, error) { - prefix := core.NewCollectionIndexKey("", "") - q, err := txn.Systemstore().Query(ctx, query.Query{ - Prefix: prefix.ToString(), - }) - if err != nil { - return nil, NewErrFailedToCreateCollectionQuery(err) - } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() - - indexes := make([]client.CollectionIndexDescription, 0) - for res := range q.Next() { - if res.Error != nil { - return nil, err - } - - var colDesk client.IndexDescription - err = json.Unmarshal(res.Value, &colDesk) - if err != nil { - return nil, NewErrInvalidStoredIndex(err) - } - indexKey, err := core.NewCollectionIndexKeyFromString(res.Key) - if err != nil { - return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) - } - indexes = append(indexes, client.CollectionIndexDescription{ - CollectionName: indexKey.CollectionName, - Index: colDesk, - }) - } - - return indexes, nil -} - -func (db *db) getCollectionIndexes( - ctx context.Context, - txn datastore.Txn, - colName string, -) ([]client.IndexDescription, error) { - prefix := core.NewCollectionIndexKey(colName, "") - q, err := txn.Systemstore().Query(ctx, query.Query{ - Prefix: prefix.ToString(), - }) - if err != nil { - return nil, NewErrFailedToCreateCollectionQuery(err) - } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() - - indexes := make([]client.IndexDescription, 0) - for res := range q.Next() { - if res.Error != nil { - return nil, err - } - - var colDesk client.IndexDescription - err = json.Unmarshal(res.Value, &colDesk) - if err != nil { - return nil, NewErrInvalidStoredIndex(err) - } - indexes = append(indexes, colDesk) - } - - return indexes, nil -} - // updateCollection updates the persisted collection description matching the name of the given // description, to the values in the given description. // @@ -788,20 +682,6 @@ func (c *collection) create(ctx context.Context, txn datastore.Txn, doc *client. return c.indexNewDoc(ctx, txn, doc) } -func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { - indexes, err := c.getIndexes(ctx, txn) - if err != nil { - return err - } - for _, index := range indexes { - err = index.Save(ctx, txn, doc) - if err != nil { - return err - } - } - return nil -} - // Update an existing document with the new values. // Any field that needs to be removed or cleared should call doc.Clear(field) before. // Any field that is nil/empty that hasn't called Clear will be ignored. @@ -874,30 +754,6 @@ func (c *collection) Save(ctx context.Context, doc *client.Document) error { return c.commitImplicitTxn(ctx, txn) } -func (c *collection) updateIndex( - ctx context.Context, - txn datastore.Txn, - doc *client.Document, -) error { - f := new(fetcher.DocumentFetcher) - fields := make([]*client.FieldDescription, len(c.desc.Schema.Fields)) - for i, field := range c.desc.Schema.Fields { - fields[i] = &field - } - err := f.Init(&c.desc, fields, false, false) - err = err - - docKey := base.MakeDocKey(c.Description(), doc.Key().String()) - err = f.Start(ctx, txn, core.NewSpans(core.NewSpan(docKey, docKey.PrefixEnd()))) - oldDoc, err := f.FetchNextDecoded(ctx) - _, err = c.getIndexes(ctx, txn) - for _, index := range c.indexes { - err = index.Update(ctx, txn, oldDoc, doc) - } - err = f.Close() - return nil -} - func (c *collection) save( ctx context.Context, txn datastore.Txn, diff --git a/db/collection_index.go b/db/collection_index.go new file mode 100644 index 0000000000..3a8c939811 --- /dev/null +++ b/db/collection_index.go @@ -0,0 +1,167 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package db + +import ( + "context" + "encoding/json" + + "github.com/ipfs/go-datastore/query" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/core" + "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/db/base" + "github.com/sourcenetwork/defradb/db/fetcher" +) + +// createCollectionIndex creates a new collection index and saves it to the database in its system store. +func (db *db) createCollectionIndex( + ctx context.Context, + txn datastore.Txn, + collectionName string, + desc client.IndexDescription, +) (client.IndexDescription, error) { + col, err := db.getCollectionByName(ctx, txn, collectionName) + if err != nil { + return client.IndexDescription{}, NewErrCollectionDoesntExist(collectionName) + } + col = col.WithTxn(txn) + return col.CreateIndex(ctx, desc) +} + +func (db *db) dropCollectionIndex( + ctx context.Context, + txn datastore.Txn, + collectionName, indexName string, +) error { + col, err := db.getCollectionByName(ctx, txn, collectionName) + if err != nil { + return NewErrCollectionDoesntExist(collectionName) + } + col = col.WithTxn(txn) + return col.DropIndex(ctx, indexName) +} + +// getAllCollectionIndexes returns all the indexes in the database. +func (db *db) getAllCollectionIndexes( + ctx context.Context, + txn datastore.Txn, +) ([]client.CollectionIndexDescription, error) { + prefix := core.NewCollectionIndexKey("", "") + q, err := txn.Systemstore().Query(ctx, query.Query{ + Prefix: prefix.ToString(), + }) + if err != nil { + return nil, NewErrFailedToCreateCollectionQuery(err) + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + indexes := make([]client.CollectionIndexDescription, 0) + for res := range q.Next() { + if res.Error != nil { + return nil, err + } + + var colDesk client.IndexDescription + err = json.Unmarshal(res.Value, &colDesk) + if err != nil { + return nil, NewErrInvalidStoredIndex(err) + } + indexKey, err := core.NewCollectionIndexKeyFromString(res.Key) + if err != nil { + return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) + } + indexes = append(indexes, client.CollectionIndexDescription{ + CollectionName: indexKey.CollectionName, + Index: colDesk, + }) + } + + return indexes, nil +} + +func (db *db) getCollectionIndexes( + ctx context.Context, + txn datastore.Txn, + colName string, +) ([]client.IndexDescription, error) { + prefix := core.NewCollectionIndexKey(colName, "") + q, err := txn.Systemstore().Query(ctx, query.Query{ + Prefix: prefix.ToString(), + }) + if err != nil { + return nil, NewErrFailedToCreateCollectionQuery(err) + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + indexes := make([]client.IndexDescription, 0) + for res := range q.Next() { + if res.Error != nil { + return nil, err + } + + var colDesk client.IndexDescription + err = json.Unmarshal(res.Value, &colDesk) + if err != nil { + return nil, NewErrInvalidStoredIndex(err) + } + indexes = append(indexes, colDesk) + } + + return indexes, nil +} + +func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { + indexes, err := c.getIndexes(ctx, txn) + if err != nil { + return err + } + for _, index := range indexes { + err = index.Save(ctx, txn, doc) + if err != nil { + return err + } + } + return nil +} + +func (c *collection) updateIndex( + ctx context.Context, + txn datastore.Txn, + doc *client.Document, +) error { + f := new(fetcher.DocumentFetcher) + fields := make([]*client.FieldDescription, len(c.desc.Schema.Fields)) + for i, field := range c.desc.Schema.Fields { + fields[i] = &field + } + err := f.Init(&c.desc, fields, false, false) + err = err + + docKey := base.MakeDocKey(c.Description(), doc.Key().String()) + err = f.Start(ctx, txn, core.NewSpans(core.NewSpan(docKey, docKey.PrefixEnd()))) + oldDoc, err := f.FetchNextDecoded(ctx) + _, err = c.getIndexes(ctx, txn) + for _, index := range c.indexes { + err = index.Update(ctx, txn, oldDoc, doc) + } + err = f.Close() + return nil +} From bedd748d487696c35d1c296d6c16b50bc11f8d5c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 29 May 2023 10:36:38 +0200 Subject: [PATCH 056/120] Add EncodedDocument interface to fetcher. Create FetcherMock --- db/fetcher/encoded_doc.go | 28 ++- db/fetcher/fetcher.go | 13 +- db/fetcher/mocks/EncodedDocument.go | 219 ++++++++++++++++++ db/fetcher/mocks/Fetcher.go | 345 ++++++++++++++++++++++++++++ db/fetcher/mocks/utils.go | 21 ++ 5 files changed, 614 insertions(+), 12 deletions(-) create mode 100644 db/fetcher/mocks/EncodedDocument.go create mode 100644 db/fetcher/mocks/Fetcher.go create mode 100644 db/fetcher/mocks/utils.go diff --git a/db/fetcher/encoded_doc.go b/db/fetcher/encoded_doc.go index a141c50652..36a77ccb46 100644 --- a/db/fetcher/encoded_doc.go +++ b/db/fetcher/encoded_doc.go @@ -20,6 +20,18 @@ import ( "github.com/sourcenetwork/defradb/core" ) +type EncodedDocument interface { + // Key returns the key of the document + Key() []byte + // Reset re-initializes the EncodedDocument object. + Reset(newKey []byte) + // Decode returns a properly decoded document object + Decode() (*client.Document, error) + // DecodeToDoc returns a decoded document as a + // map of field/value pairs + DecodeToDoc(*core.DocumentMapping) (core.Doc, error) +} + type EPTuple []encProperty // EncProperty is an encoded property of a EncodedDocument @@ -178,19 +190,25 @@ func convertToInt(propertyName string, untypedValue any) (int64, error) { // @todo: Implement Encoded Document type type encodedDocument struct { - Key []byte + key []byte Properties map[client.FieldDescription]*encProperty } +var _ EncodedDocument = (*encodedDocument)(nil) + +func (encdoc *encodedDocument) Key() []byte { + return encdoc.key +} + // Reset re-initializes the EncodedDocument object. -func (encdoc *encodedDocument) Reset() { +func (encdoc *encodedDocument) Reset(newKey []byte) { encdoc.Properties = make(map[client.FieldDescription]*encProperty) - encdoc.Key = nil + encdoc.key = newKey } // Decode returns a properly decoded document object func (encdoc *encodedDocument) Decode() (*client.Document, error) { - key, err := client.NewDocKeyFromString(string(encdoc.Key)) + key, err := client.NewDocKeyFromString(string(encdoc.key)) if err != nil { return nil, err } @@ -213,7 +231,7 @@ func (encdoc *encodedDocument) Decode() (*client.Document, error) { // map of field/value pairs func (encdoc *encodedDocument) DecodeToDoc(mapping *core.DocumentMapping) (core.Doc, error) { doc := mapping.NewDoc() - doc.SetKey(string(encdoc.Key)) + doc.SetKey(string(encdoc.key)) for fieldDesc, prop := range encdoc.Properties { _, val, err := prop.Decode() if err != nil { diff --git a/db/fetcher/fetcher.go b/db/fetcher/fetcher.go index 5de2a8899e..b5ac74c330 100644 --- a/db/fetcher/fetcher.go +++ b/db/fetcher/fetcher.go @@ -28,7 +28,7 @@ import ( type Fetcher interface { Init(col *client.CollectionDescription, fields []*client.FieldDescription, reverse bool, showDeleted bool) error Start(ctx context.Context, txn datastore.Txn, spans core.Spans) error - FetchNext(ctx context.Context) (*encodedDocument, error) + FetchNext(ctx context.Context) (EncodedDocument, error) FetchNextDecoded(ctx context.Context) (*client.Document, error) FetchNextDoc(ctx context.Context, mapping *core.DocumentMapping) ([]byte, core.Doc, error) Close() error @@ -267,7 +267,7 @@ func (df *DocumentFetcher) nextKey(ctx context.Context) (spanDone bool, err erro } // check if we've crossed document boundries - if df.doc.Key != nil && df.kv.Key.DocKey != string(df.doc.Key) { + if df.doc.Key() != nil && df.kv.Key.DocKey != string(df.doc.Key()) { df.isReadingDocument = false return true, nil } @@ -315,8 +315,7 @@ func (df *DocumentFetcher) processKV(kv *core.KeyValue) error { if !df.isReadingDocument { df.isReadingDocument = true - df.doc.Reset() - df.doc.Key = []byte(kv.Key.DocKey) + df.doc.Reset([]byte(kv.Key.DocKey)) } // we have to skip the object marker @@ -349,7 +348,7 @@ func (df *DocumentFetcher) processKV(kv *core.KeyValue) error { // FetchNext returns a raw binary encoded document. It iterates over all the relevant // keypairs from the underlying store and constructs the document. -func (df *DocumentFetcher) FetchNext(ctx context.Context) (*encodedDocument, error) { +func (df *DocumentFetcher) FetchNext(ctx context.Context) (EncodedDocument, error) { if df.kvEnd { return nil, nil } @@ -413,7 +412,7 @@ func (df *DocumentFetcher) FetchNextDoc( mapping *core.DocumentMapping, ) ([]byte, core.Doc, error) { var err error - var encdoc *encodedDocument + var encdoc EncodedDocument var status client.DocumentStatus // If the deletedDocFetcher isn't nil, this means that the user requested to include the deleted documents @@ -461,7 +460,7 @@ func (df *DocumentFetcher) FetchNextDoc( return nil, core.Doc{}, err } doc.Status = status - return encdoc.Key, doc, err + return encdoc.Key(), doc, err } // Close closes the DocumentFetcher. diff --git a/db/fetcher/mocks/EncodedDocument.go b/db/fetcher/mocks/EncodedDocument.go new file mode 100644 index 0000000000..d6b88fb3c9 --- /dev/null +++ b/db/fetcher/mocks/EncodedDocument.go @@ -0,0 +1,219 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + client "github.com/sourcenetwork/defradb/client" + core "github.com/sourcenetwork/defradb/core" + + mock "github.com/stretchr/testify/mock" +) + +// EncodedDocument is an autogenerated mock type for the EncodedDocument type +type EncodedDocument struct { + mock.Mock +} + +type EncodedDocument_Expecter struct { + mock *mock.Mock +} + +func (_m *EncodedDocument) EXPECT() *EncodedDocument_Expecter { + return &EncodedDocument_Expecter{mock: &_m.Mock} +} + +// Decode provides a mock function with given fields: +func (_m *EncodedDocument) Decode() (*client.Document, error) { + ret := _m.Called() + + var r0 *client.Document + var r1 error + if rf, ok := ret.Get(0).(func() (*client.Document, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *client.Document); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*client.Document) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EncodedDocument_Decode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Decode' +type EncodedDocument_Decode_Call struct { + *mock.Call +} + +// Decode is a helper method to define mock.On call +func (_e *EncodedDocument_Expecter) Decode() *EncodedDocument_Decode_Call { + return &EncodedDocument_Decode_Call{Call: _e.mock.On("Decode")} +} + +func (_c *EncodedDocument_Decode_Call) Run(run func()) *EncodedDocument_Decode_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *EncodedDocument_Decode_Call) Return(_a0 *client.Document, _a1 error) *EncodedDocument_Decode_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *EncodedDocument_Decode_Call) RunAndReturn(run func() (*client.Document, error)) *EncodedDocument_Decode_Call { + _c.Call.Return(run) + return _c +} + +// DecodeToDoc provides a mock function with given fields: _a0 +func (_m *EncodedDocument) DecodeToDoc(_a0 *core.DocumentMapping) (core.Doc, error) { + ret := _m.Called(_a0) + + var r0 core.Doc + var r1 error + if rf, ok := ret.Get(0).(func(*core.DocumentMapping) (core.Doc, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(*core.DocumentMapping) core.Doc); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(core.Doc) + } + + if rf, ok := ret.Get(1).(func(*core.DocumentMapping) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EncodedDocument_DecodeToDoc_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DecodeToDoc' +type EncodedDocument_DecodeToDoc_Call struct { + *mock.Call +} + +// DecodeToDoc is a helper method to define mock.On call +// - _a0 *core.DocumentMapping +func (_e *EncodedDocument_Expecter) DecodeToDoc(_a0 interface{}) *EncodedDocument_DecodeToDoc_Call { + return &EncodedDocument_DecodeToDoc_Call{Call: _e.mock.On("DecodeToDoc", _a0)} +} + +func (_c *EncodedDocument_DecodeToDoc_Call) Run(run func(_a0 *core.DocumentMapping)) *EncodedDocument_DecodeToDoc_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*core.DocumentMapping)) + }) + return _c +} + +func (_c *EncodedDocument_DecodeToDoc_Call) Return(_a0 core.Doc, _a1 error) *EncodedDocument_DecodeToDoc_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *EncodedDocument_DecodeToDoc_Call) RunAndReturn(run func(*core.DocumentMapping) (core.Doc, error)) *EncodedDocument_DecodeToDoc_Call { + _c.Call.Return(run) + return _c +} + +// Key provides a mock function with given fields: +func (_m *EncodedDocument) Key() []byte { + ret := _m.Called() + + var r0 []byte + if rf, ok := ret.Get(0).(func() []byte); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + return r0 +} + +// EncodedDocument_Key_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Key' +type EncodedDocument_Key_Call struct { + *mock.Call +} + +// Key is a helper method to define mock.On call +func (_e *EncodedDocument_Expecter) Key() *EncodedDocument_Key_Call { + return &EncodedDocument_Key_Call{Call: _e.mock.On("Key")} +} + +func (_c *EncodedDocument_Key_Call) Run(run func()) *EncodedDocument_Key_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *EncodedDocument_Key_Call) Return(_a0 []byte) *EncodedDocument_Key_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EncodedDocument_Key_Call) RunAndReturn(run func() []byte) *EncodedDocument_Key_Call { + _c.Call.Return(run) + return _c +} + +// Reset provides a mock function with given fields: newKey +func (_m *EncodedDocument) Reset(newKey []byte) { + _m.Called(newKey) +} + +// EncodedDocument_Reset_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Reset' +type EncodedDocument_Reset_Call struct { + *mock.Call +} + +// Reset is a helper method to define mock.On call +// - newKey []byte +func (_e *EncodedDocument_Expecter) Reset(newKey interface{}) *EncodedDocument_Reset_Call { + return &EncodedDocument_Reset_Call{Call: _e.mock.On("Reset", newKey)} +} + +func (_c *EncodedDocument_Reset_Call) Run(run func(newKey []byte)) *EncodedDocument_Reset_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]byte)) + }) + return _c +} + +func (_c *EncodedDocument_Reset_Call) Return() *EncodedDocument_Reset_Call { + _c.Call.Return() + return _c +} + +func (_c *EncodedDocument_Reset_Call) RunAndReturn(run func([]byte)) *EncodedDocument_Reset_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewEncodedDocument interface { + mock.TestingT + Cleanup(func()) +} + +// NewEncodedDocument creates a new instance of EncodedDocument. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEncodedDocument(t mockConstructorTestingTNewEncodedDocument) *EncodedDocument { + mock := &EncodedDocument{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/db/fetcher/mocks/Fetcher.go b/db/fetcher/mocks/Fetcher.go new file mode 100644 index 0000000000..3fa8c60fb7 --- /dev/null +++ b/db/fetcher/mocks/Fetcher.go @@ -0,0 +1,345 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + client "github.com/sourcenetwork/defradb/client" + + core "github.com/sourcenetwork/defradb/core" + + datastore "github.com/sourcenetwork/defradb/datastore" + + fetcher "github.com/sourcenetwork/defradb/db/fetcher" + + mock "github.com/stretchr/testify/mock" +) + +// Fetcher is an autogenerated mock type for the Fetcher type +type Fetcher struct { + mock.Mock +} + +type Fetcher_Expecter struct { + mock *mock.Mock +} + +func (_m *Fetcher) EXPECT() *Fetcher_Expecter { + return &Fetcher_Expecter{mock: &_m.Mock} +} + +// Close provides a mock function with given fields: +func (_m *Fetcher) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Fetcher_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type Fetcher_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +func (_e *Fetcher_Expecter) Close() *Fetcher_Close_Call { + return &Fetcher_Close_Call{Call: _e.mock.On("Close")} +} + +func (_c *Fetcher_Close_Call) Run(run func()) *Fetcher_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Fetcher_Close_Call) Return(_a0 error) *Fetcher_Close_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Fetcher_Close_Call) RunAndReturn(run func() error) *Fetcher_Close_Call { + _c.Call.Return(run) + return _c +} + +// FetchNext provides a mock function with given fields: ctx +func (_m *Fetcher) FetchNext(ctx context.Context) (fetcher.EncodedDocument, error) { + ret := _m.Called(ctx) + + var r0 fetcher.EncodedDocument + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (fetcher.EncodedDocument, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) fetcher.EncodedDocument); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(fetcher.EncodedDocument) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Fetcher_FetchNext_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchNext' +type Fetcher_FetchNext_Call struct { + *mock.Call +} + +// FetchNext is a helper method to define mock.On call +// - ctx context.Context +func (_e *Fetcher_Expecter) FetchNext(ctx interface{}) *Fetcher_FetchNext_Call { + return &Fetcher_FetchNext_Call{Call: _e.mock.On("FetchNext", ctx)} +} + +func (_c *Fetcher_FetchNext_Call) Run(run func(ctx context.Context)) *Fetcher_FetchNext_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Fetcher_FetchNext_Call) Return(_a0 fetcher.EncodedDocument, _a1 error) *Fetcher_FetchNext_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Fetcher_FetchNext_Call) RunAndReturn(run func(context.Context) (fetcher.EncodedDocument, error)) *Fetcher_FetchNext_Call { + _c.Call.Return(run) + return _c +} + +// FetchNextDecoded provides a mock function with given fields: ctx +func (_m *Fetcher) FetchNextDecoded(ctx context.Context) (*client.Document, error) { + ret := _m.Called(ctx) + + var r0 *client.Document + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*client.Document, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *client.Document); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*client.Document) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Fetcher_FetchNextDecoded_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchNextDecoded' +type Fetcher_FetchNextDecoded_Call struct { + *mock.Call +} + +// FetchNextDecoded is a helper method to define mock.On call +// - ctx context.Context +func (_e *Fetcher_Expecter) FetchNextDecoded(ctx interface{}) *Fetcher_FetchNextDecoded_Call { + return &Fetcher_FetchNextDecoded_Call{Call: _e.mock.On("FetchNextDecoded", ctx)} +} + +func (_c *Fetcher_FetchNextDecoded_Call) Run(run func(ctx context.Context)) *Fetcher_FetchNextDecoded_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Fetcher_FetchNextDecoded_Call) Return(_a0 *client.Document, _a1 error) *Fetcher_FetchNextDecoded_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Fetcher_FetchNextDecoded_Call) RunAndReturn(run func(context.Context) (*client.Document, error)) *Fetcher_FetchNextDecoded_Call { + _c.Call.Return(run) + return _c +} + +// FetchNextDoc provides a mock function with given fields: ctx, mapping +func (_m *Fetcher) FetchNextDoc(ctx context.Context, mapping *core.DocumentMapping) ([]byte, core.Doc, error) { + ret := _m.Called(ctx, mapping) + + var r0 []byte + var r1 core.Doc + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, *core.DocumentMapping) ([]byte, core.Doc, error)); ok { + return rf(ctx, mapping) + } + if rf, ok := ret.Get(0).(func(context.Context, *core.DocumentMapping) []byte); ok { + r0 = rf(ctx, mapping) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *core.DocumentMapping) core.Doc); ok { + r1 = rf(ctx, mapping) + } else { + r1 = ret.Get(1).(core.Doc) + } + + if rf, ok := ret.Get(2).(func(context.Context, *core.DocumentMapping) error); ok { + r2 = rf(ctx, mapping) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// Fetcher_FetchNextDoc_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchNextDoc' +type Fetcher_FetchNextDoc_Call struct { + *mock.Call +} + +// FetchNextDoc is a helper method to define mock.On call +// - ctx context.Context +// - mapping *core.DocumentMapping +func (_e *Fetcher_Expecter) FetchNextDoc(ctx interface{}, mapping interface{}) *Fetcher_FetchNextDoc_Call { + return &Fetcher_FetchNextDoc_Call{Call: _e.mock.On("FetchNextDoc", ctx, mapping)} +} + +func (_c *Fetcher_FetchNextDoc_Call) Run(run func(ctx context.Context, mapping *core.DocumentMapping)) *Fetcher_FetchNextDoc_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*core.DocumentMapping)) + }) + return _c +} + +func (_c *Fetcher_FetchNextDoc_Call) Return(_a0 []byte, _a1 core.Doc, _a2 error) *Fetcher_FetchNextDoc_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *Fetcher_FetchNextDoc_Call) RunAndReturn(run func(context.Context, *core.DocumentMapping) ([]byte, core.Doc, error)) *Fetcher_FetchNextDoc_Call { + _c.Call.Return(run) + return _c +} + +// Init provides a mock function with given fields: col, fields, reverse, showDeleted +func (_m *Fetcher) Init(col *client.CollectionDescription, fields []*client.FieldDescription, reverse bool, showDeleted bool) error { + ret := _m.Called(col, fields, reverse, showDeleted) + + var r0 error + if rf, ok := ret.Get(0).(func(*client.CollectionDescription, []*client.FieldDescription, bool, bool) error); ok { + r0 = rf(col, fields, reverse, showDeleted) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Fetcher_Init_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Init' +type Fetcher_Init_Call struct { + *mock.Call +} + +// Init is a helper method to define mock.On call +// - col *client.CollectionDescription +// - fields []*client.FieldDescription +// - reverse bool +// - showDeleted bool +func (_e *Fetcher_Expecter) Init(col interface{}, fields interface{}, reverse interface{}, showDeleted interface{}) *Fetcher_Init_Call { + return &Fetcher_Init_Call{Call: _e.mock.On("Init", col, fields, reverse, showDeleted)} +} + +func (_c *Fetcher_Init_Call) Run(run func(col *client.CollectionDescription, fields []*client.FieldDescription, reverse bool, showDeleted bool)) *Fetcher_Init_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*client.CollectionDescription), args[1].([]*client.FieldDescription), args[2].(bool), args[3].(bool)) + }) + return _c +} + +func (_c *Fetcher_Init_Call) Return(_a0 error) *Fetcher_Init_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Fetcher_Init_Call) RunAndReturn(run func(*client.CollectionDescription, []*client.FieldDescription, bool, bool) error) *Fetcher_Init_Call { + _c.Call.Return(run) + return _c +} + +// Start provides a mock function with given fields: ctx, txn, spans +func (_m *Fetcher) Start(ctx context.Context, txn datastore.Txn, spans core.Spans) error { + ret := _m.Called(ctx, txn, spans) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, datastore.Txn, core.Spans) error); ok { + r0 = rf(ctx, txn, spans) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Fetcher_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' +type Fetcher_Start_Call struct { + *mock.Call +} + +// Start is a helper method to define mock.On call +// - ctx context.Context +// - txn datastore.Txn +// - spans core.Spans +func (_e *Fetcher_Expecter) Start(ctx interface{}, txn interface{}, spans interface{}) *Fetcher_Start_Call { + return &Fetcher_Start_Call{Call: _e.mock.On("Start", ctx, txn, spans)} +} + +func (_c *Fetcher_Start_Call) Run(run func(ctx context.Context, txn datastore.Txn, spans core.Spans)) *Fetcher_Start_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(datastore.Txn), args[2].(core.Spans)) + }) + return _c +} + +func (_c *Fetcher_Start_Call) Return(_a0 error) *Fetcher_Start_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Fetcher_Start_Call) RunAndReturn(run func(context.Context, datastore.Txn, core.Spans) error) *Fetcher_Start_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewFetcher interface { + mock.TestingT + Cleanup(func()) +} + +// NewFetcher creates a new instance of Fetcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewFetcher(t mockConstructorTestingTNewFetcher) *Fetcher { + mock := &Fetcher{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/db/fetcher/mocks/utils.go b/db/fetcher/mocks/utils.go new file mode 100644 index 0000000000..e28b9ca1a8 --- /dev/null +++ b/db/fetcher/mocks/utils.go @@ -0,0 +1,21 @@ +package mocks + +import ( + "testing" + + client "github.com/sourcenetwork/defradb/client" + core "github.com/sourcenetwork/defradb/core" + mock "github.com/stretchr/testify/mock" +) + +func NewStubbedFetcher(t *testing.T) *Fetcher { + f := NewFetcher(t) + f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil) + f.EXPECT().Start(mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil) + f.EXPECT().FetchNext(mock.Anything).Maybe().Return(nil, nil) + f.EXPECT().FetchNextDoc(mock.Anything, mock.Anything).Maybe(). + Return(NewEncodedDocument(t), core.Doc{}, nil) + f.EXPECT().FetchNextDecoded(mock.Anything).Maybe().Return(&client.Document{}, nil) + f.EXPECT().Close().Maybe().Return(nil) + return f +} From ee253ee16240e364c97ee3f3f358a1e5f3e4f797 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 29 May 2023 10:37:36 +0200 Subject: [PATCH 057/120] Check if fetcher returns errors --- db/collection.go | 6 ++-- db/collection_index.go | 20 ++++++++++-- db/index_test.go | 6 ++-- db/indexed_docs_test.go | 71 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/db/collection.go b/db/collection.go index 3fcab67986..58f107cb24 100644 --- a/db/collection.go +++ b/db/collection.go @@ -30,6 +30,7 @@ import ( "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/db/base" + "github.com/sourcenetwork/defradb/db/fetcher" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/logging" @@ -57,8 +58,9 @@ type collection struct { desc client.CollectionDescription - isIndexCached bool - indexes []CollectionIndex + isIndexCached bool + indexes []CollectionIndex + fetcherFactory func() fetcher.Fetcher } // @todo: Move the base Descriptions to an internal API within the db/ package. diff --git a/db/collection_index.go b/db/collection_index.go index 3a8c939811..d48a92ee0e 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -147,21 +147,37 @@ func (c *collection) updateIndex( txn datastore.Txn, doc *client.Document, ) error { - f := new(fetcher.DocumentFetcher) + var f fetcher.Fetcher + if c.fetcherFactory != nil { + f = c.fetcherFactory() + } else { + f = new(fetcher.DocumentFetcher) + } fields := make([]*client.FieldDescription, len(c.desc.Schema.Fields)) for i, field := range c.desc.Schema.Fields { fields[i] = &field } err := f.Init(&c.desc, fields, false, false) - err = err + if err != nil { + return err + } docKey := base.MakeDocKey(c.Description(), doc.Key().String()) err = f.Start(ctx, txn, core.NewSpans(core.NewSpan(docKey, docKey.PrefixEnd()))) + if err != nil { + return err + } oldDoc, err := f.FetchNextDecoded(ctx) + if err != nil { + return err + } _, err = c.getIndexes(ctx, txn) for _, index := range c.indexes { err = index.Update(ctx, txn, oldDoc, doc) } err = f.Close() + if err != nil { + return err + } return nil } diff --git a/db/index_test.go b/db/index_test.go index a9ac2c6f15..26cb7b08cd 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -54,7 +54,7 @@ type indexTestFixture struct { ctx context.Context db *implicitTxnDB txn datastore.Txn - users client.Collection + users *collection t *testing.T } @@ -259,14 +259,14 @@ func (f *indexTestFixture) getCollectionIndexes(colName string) ([]client.IndexD func (f *indexTestFixture) createCollection( desc client.CollectionDescription, -) client.Collection { +) *collection { col, err := f.db.createCollection(f.ctx, f.txn, desc) assert.NoError(f.t, err) err = f.txn.Commit(f.ctx) assert.NoError(f.t, err) f.txn, err = f.db.NewTxn(f.ctx, false) assert.NoError(f.t, err) - return col + return col.(*collection) } func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 29f188ca48..56479da4b0 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -12,6 +12,8 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore/mocks" + "github.com/sourcenetwork/defradb/db/fetcher" + fetcherMocks "github.com/sourcenetwork/defradb/db/fetcher/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -644,3 +646,72 @@ func TestNonUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { require.NoError(t, err) } } + +func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { + testError := errors.New("test error") + + cases := []struct { + Name string + PrepareFetcher func() fetcher.Fetcher + }{ + { + Name: "Fails to init", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Unset() + f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testError) + return f + }, + }, + { + Name: "Fails to start", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().Start(mock.Anything, mock.Anything, mock.Anything).Unset() + f.EXPECT().Start(mock.Anything, mock.Anything, mock.Anything).Return(testError) + return f + }, + }, + { + Name: "Fails to fetch next decoded", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().FetchNextDecoded(mock.Anything).Unset() + f.EXPECT().FetchNextDecoded(mock.Anything).Return(nil, testError) + return f + }, + }, + { + Name: "Fails to close", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(testError) + return f + }, + }, + } + + for _, tc := range cases { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + + f.users.fetcherFactory = tc.PrepareFetcher + oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + + err := doc.Set(usersNameFieldName, "Islam") + require.NoError(t, err, tc.Name) + err = f.users.Update(f.ctx, doc) + require.Error(t, err, tc.Name) + + newKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + + _, err = f.txn.Datastore().Get(f.ctx, oldKey.ToDS()) + require.NoError(t, err, tc.Name) + _, err = f.txn.Datastore().Get(f.ctx, newKey.ToDS()) + require.Error(t, err, tc.Name) + } +} From f9f2b47a40134d638784a123b48ee76f48458d2e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 29 May 2023 13:58:43 +0200 Subject: [PATCH 058/120] Check more errors on index update --- db/collection_index.go | 14 ++++++++++---- db/index.go | 4 +++- db/indexed_docs_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index d48a92ee0e..1f7f91d7d4 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -159,25 +159,31 @@ func (c *collection) updateIndex( } err := f.Init(&c.desc, fields, false, false) if err != nil { + _ = f.Close() return err } docKey := base.MakeDocKey(c.Description(), doc.Key().String()) err = f.Start(ctx, txn, core.NewSpans(core.NewSpan(docKey, docKey.PrefixEnd()))) if err != nil { + _ = f.Close() return err } oldDoc, err := f.FetchNextDecoded(ctx) if err != nil { + _ = f.Close() return err } - _, err = c.getIndexes(ctx, txn) - for _, index := range c.indexes { - err = index.Update(ctx, txn, oldDoc, doc) - } err = f.Close() if err != nil { return err } + _, err = c.getIndexes(ctx, txn) + for _, index := range c.indexes { + err = index.Update(ctx, txn, oldDoc, doc) + if err != nil { + return err + } + } return nil } diff --git a/db/index.go b/db/index.go index d1e150250d..2170a97608 100644 --- a/db/index.go +++ b/db/index.go @@ -152,7 +152,9 @@ func (i *collectionSimpleIndex) Update( newDoc *client.Document, ) error { key, err := i.getDocKey(oldDoc) - err = err + if err != nil { + return err + } err = txn.Datastore().Delete(ctx, key.ToDS()) i.Save(ctx, txn, newDoc) return nil diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 56479da4b0..5aff2f2a83 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -660,6 +660,8 @@ func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { f := fetcherMocks.NewStubbedFetcher(t) f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Unset() f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testError) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(nil) return f }, }, @@ -669,6 +671,8 @@ func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { f := fetcherMocks.NewStubbedFetcher(t) f.EXPECT().Start(mock.Anything, mock.Anything, mock.Anything).Unset() f.EXPECT().Start(mock.Anything, mock.Anything, mock.Anything).Return(testError) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(nil) return f }, }, @@ -678,6 +682,8 @@ func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { f := fetcherMocks.NewStubbedFetcher(t) f.EXPECT().FetchNextDecoded(mock.Anything).Unset() f.EXPECT().FetchNextDecoded(mock.Anything).Return(nil, testError) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(nil) return f }, }, @@ -715,3 +721,27 @@ func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { require.Error(t, err, tc.Name) } } + +func TestNonUniqueUpdate_IfFailsToUpdateIndex_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnAge() + + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + f.commitTxn() + + validKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() + invalidKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc). + Values("invalid").Build() + + err := f.txn.Datastore().Delete(f.ctx, validKey.ToDS()) + require.NoError(f.t, err) + err = f.txn.Datastore().Put(f.ctx, invalidKey.ToDS(), []byte{}) + require.NoError(f.t, err) + f.commitTxn() + + err = doc.Set(usersAgeFieldName, 23) + require.NoError(t, err) + err = f.users.Update(f.ctx, doc) + require.Error(t, err) +} From 73204448d4ff99e140a29a9c6618855a26778ea8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 29 May 2023 21:59:06 +0200 Subject: [PATCH 059/120] Pass relevant fields to fetcher --- db/collection_index.go | 30 ++++++++++++++++++++++++------ db/indexed_docs_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index 1f7f91d7d4..54a7e18a86 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -142,22 +142,41 @@ func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *cl return nil } +func (c *collection) collectIndexedFields() []*client.FieldDescription { + fieldsMap := make(map[string]*client.FieldDescription) + for _, index := range c.indexes { + for _, field := range index.Description().Fields { + for _, colField := range c.desc.Schema.Fields { + if field.Name == colField.Name { + fieldsMap[field.Name] = &colField + break + } + } + } + } + fields := make([]*client.FieldDescription, 0, len(fieldsMap)) + for _, field := range fieldsMap { + fields = append(fields, field) + } + return fields +} + func (c *collection) updateIndex( ctx context.Context, txn datastore.Txn, doc *client.Document, ) error { + _, err := c.getIndexes(ctx, txn) + if err != nil { + return err + } var f fetcher.Fetcher if c.fetcherFactory != nil { f = c.fetcherFactory() } else { f = new(fetcher.DocumentFetcher) } - fields := make([]*client.FieldDescription, len(c.desc.Schema.Fields)) - for i, field := range c.desc.Schema.Fields { - fields[i] = &field - } - err := f.Init(&c.desc, fields, false, false) + err = f.Init(&c.desc, c.collectIndexedFields(), false, false) if err != nil { _ = f.Close() return err @@ -178,7 +197,6 @@ func (c *collection) updateIndex( if err != nil { return err } - _, err = c.getIndexes(ctx, txn) for _, index := range c.indexes { err = index.Update(ctx, txn, oldDoc, doc) if err != nil { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 5aff2f2a83..3d1353fb3a 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -745,3 +745,33 @@ func TestNonUniqueUpdate_IfFailsToUpdateIndex_ReturnError(t *testing.T) { err = f.users.Update(f.ctx, doc) require.Error(t, err) } + +func TestNonUniqueUpdate_ShouldPassToFetcherOnlyRelevantFields(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + f.createUserCollectionIndexOnAge() + + f.users.fetcherFactory = func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Unset() + f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything). + RunAndReturn(func( + col *client.CollectionDescription, + fields []*client.FieldDescription, + reverse, showDeleted bool, + ) error { + require.Equal(t, 2, len(fields)) + require.ElementsMatch(t, + []string{usersNameFieldName, usersAgeFieldName}, + []string{fields[0].Name, fields[1].Name}) + return errors.New("early exit") + }) + return f + } + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + + err := doc.Set(usersNameFieldName, "Islam") + require.NoError(t, err) + _ = f.users.Update(f.ctx, doc) +} From b7c04d9945f810c846a862e95acb70eaf500e2d7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 29 May 2023 22:11:20 +0200 Subject: [PATCH 060/120] Use existing method to fetch a doc --- db/collection_get.go | 12 +++++++++--- db/collection_index.go | 27 +-------------------------- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/db/collection_get.go b/db/collection_get.go index 678a154598..7225d1215a 100644 --- a/db/collection_get.go +++ b/db/collection_get.go @@ -37,7 +37,7 @@ func (c *collection) Get(ctx context.Context, key client.DocKey, showDeleted boo return nil, client.ErrDocumentNotFound } - doc, err := c.get(ctx, txn, dsKey, showDeleted) + doc, err := c.get(ctx, txn, dsKey, nil, showDeleted) if err != nil { return nil, err } @@ -48,13 +48,19 @@ func (c *collection) get( ctx context.Context, txn datastore.Txn, key core.PrimaryDataStoreKey, + fields []*client.FieldDescription, showDeleted bool, ) (*client.Document, error) { // create a new document fetcher - df := new(fetcher.DocumentFetcher) + var df fetcher.Fetcher + if c.fetcherFactory != nil { + df = c.fetcherFactory() + } else { + df = new(fetcher.DocumentFetcher) + } desc := &c.desc // initialize it with the primary index - err := df.Init(&c.desc, nil, false, showDeleted) + err := df.Init(&c.desc, fields, false, showDeleted) if err != nil { _ = df.Close() return nil, err diff --git a/db/collection_index.go b/db/collection_index.go index 54a7e18a86..e9c2167788 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -19,8 +19,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/db/base" - "github.com/sourcenetwork/defradb/db/fetcher" ) // createCollectionIndex creates a new collection index and saves it to the database in its system store. @@ -170,30 +168,7 @@ func (c *collection) updateIndex( if err != nil { return err } - var f fetcher.Fetcher - if c.fetcherFactory != nil { - f = c.fetcherFactory() - } else { - f = new(fetcher.DocumentFetcher) - } - err = f.Init(&c.desc, c.collectIndexedFields(), false, false) - if err != nil { - _ = f.Close() - return err - } - - docKey := base.MakeDocKey(c.Description(), doc.Key().String()) - err = f.Start(ctx, txn, core.NewSpans(core.NewSpan(docKey, docKey.PrefixEnd()))) - if err != nil { - _ = f.Close() - return err - } - oldDoc, err := f.FetchNextDecoded(ctx) - if err != nil { - _ = f.Close() - return err - } - err = f.Close() + oldDoc, err := c.get(ctx, txn, c.getPrimaryKeyFromDocKey(doc.Key()), c.collectIndexedFields(), false) if err != nil { return err } From f52ba4a2271d43a5a3f2071c3695fc6a23418555 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 May 2023 12:05:10 +0200 Subject: [PATCH 061/120] Check more errors --- db/collection.go | 15 ++++++------ db/index.go | 6 +++-- db/indexed_docs_test.go | 53 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/db/collection.go b/db/collection.go index 58f107cb24..d66cb3c7b5 100644 --- a/db/collection.go +++ b/db/collection.go @@ -574,13 +574,14 @@ func (c *collection) SchemaID() string { // handle instead of a raw DB handle. func (c *collection) WithTxn(txn datastore.Txn) client.Collection { return &collection{ - db: c.db, - txn: immutable.Some(txn), - desc: c.desc, - colID: c.colID, - schemaID: c.schemaID, - isIndexCached: c.isIndexCached, - indexes: c.indexes, + db: c.db, + txn: immutable.Some(txn), + desc: c.desc, + colID: c.colID, + schemaID: c.schemaID, + isIndexCached: c.isIndexCached, + indexes: c.indexes, + fetcherFactory: c.fetcherFactory, } } diff --git a/db/index.go b/db/index.go index 2170a97608..4b2209cf25 100644 --- a/db/index.go +++ b/db/index.go @@ -156,8 +156,10 @@ func (i *collectionSimpleIndex) Update( return err } err = txn.Datastore().Delete(ctx, key.ToDS()) - i.Save(ctx, txn, newDoc) - return nil + if err != nil { + return err + } + return i.Save(ctx, txn, newDoc) } func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn) error { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 3d1353fb3a..a45e9262b6 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -775,3 +775,56 @@ func TestNonUniqueUpdate_ShouldPassToFetcherOnlyRelevantFields(t *testing.T) { require.NoError(t, err) _ = f.users.Update(f.ctx, doc) } + +func TestNonUniqueUpdate_IfDatastoreFails_ReturnError(t *testing.T) { + testErr := errors.New("error") + + cases := []struct { + Name string + StubDataStore func(*mocks.DSReaderWriter_Expecter) + }{ + { + Name: "Delete old value", + StubDataStore: func(ds *mocks.DSReaderWriter_Expecter) { + ds.Delete(mock.Anything, mock.Anything).Return(testErr) + ds.Get(mock.Anything, mock.Anything).Maybe().Return([]byte{}, nil) + }, + }, + { + Name: "Store new value", + StubDataStore: func(ds *mocks.DSReaderWriter_Expecter) { + ds.Delete(mock.Anything, mock.Anything).Maybe().Return(nil) + ds.Get(mock.Anything, mock.Anything).Maybe().Return([]byte{}, nil) + ds.Put(mock.Anything, mock.Anything, mock.Anything).Maybe().Return(testErr) + }, + }, + } + + for _, tc := range cases { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + doc := f.newUserDoc("John", 21) + + f.users.fetcherFactory = func() fetcher.Fetcher { + df := fetcherMocks.NewStubbedFetcher(t) + df.EXPECT().FetchNextDecoded(mock.Anything).Unset() + df.EXPECT().FetchNextDecoded(mock.Anything).Return(doc, nil) + return df + } + + f.saveDocToCollection(doc, f.users) + + err := doc.Set(usersNameFieldName, "Islam") + require.NoError(t, err) + + mockedTxn := f.mockTxn() + mockedTxn.MockDatastore = mocks.NewDSReaderWriter(f.t) + tc.StubDataStore(mockedTxn.MockDatastore.EXPECT()) + mockedTxn.EXPECT().Datastore().Unset() + mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore).Maybe() + + err = f.users.WithTxn(mockedTxn).Update(f.ctx, doc) + require.ErrorIs(t, err, testErr) + } +} From ea959b4503b881a325f62fd96b621e790be6d33b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 May 2023 13:22:45 +0200 Subject: [PATCH 062/120] Include IndexDescription into CollectionDescription --- client/descriptions.go | 3 +++ core/parser.go | 8 ++----- db/schema.go | 2 +- request/graphql/parser.go | 1 - request/graphql/schema/collection.go | 25 +++++++++------------ request/graphql/schema/descriptions_test.go | 15 ++++++++++++- request/graphql/schema/index_test.go | 10 ++++----- 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/client/descriptions.go b/client/descriptions.go index cd1d7fc53d..fbba85fd35 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -29,6 +29,9 @@ type CollectionDescription struct { // Schema contains the data type information that this Collection uses. Schema SchemaDescription + + // Indexes contains the indexes that this Collection has. + Indexes []IndexDescription } // IDString returns the collection ID as a string. diff --git a/core/parser.go b/core/parser.go index b7c567cf26..ee2d2cfbf1 100644 --- a/core/parser.go +++ b/core/parser.go @@ -50,12 +50,8 @@ type Parser interface { // NewFilterFromString creates a new filter from a string. NewFilterFromString(collectionType string, body string) (immutable.Option[request.Filter], error) - // ParseSDL parses an SDL string into a set of collection descriptions and indexes. - ParseSDL(ctx context.Context, schemaString string) ( - []client.CollectionDescription, - [][]client.IndexDescription, - error, - ) + // ParseSDL parses an SDL string into a set of collection descriptions. + ParseSDL(ctx context.Context, schemaString string) ([]client.CollectionDescription, error) // Adds the given schema to this parser's model. SetSchema(ctx context.Context, txn datastore.Txn, collections []client.CollectionDescription) error diff --git a/db/schema.go b/db/schema.go index 83765994b4..e85b0b6a72 100644 --- a/db/schema.go +++ b/db/schema.go @@ -33,7 +33,7 @@ func (db *db) addSchema( return nil, err } - newDescriptions, _, err := db.parser.ParseSDL(ctx, schemaString) + newDescriptions, err := db.parser.ParseSDL(ctx, schemaString) if err != nil { return nil, err } diff --git a/request/graphql/parser.go b/request/graphql/parser.go index f6a9d19425..ddd13d9e62 100644 --- a/request/graphql/parser.go +++ b/request/graphql/parser.go @@ -105,7 +105,6 @@ func (p *parser) Parse(ast *ast.Document) (*request.Request, []error) { func (p *parser) ParseSDL(ctx context.Context, schemaString string) ( []client.CollectionDescription, - [][]client.IndexDescription, error, ) { return schema.FromString(ctx, schemaString) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index ef55177156..4f774b41d0 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -26,7 +26,6 @@ import ( // FromString parses a GQL SDL string into a set of collection descriptions. func FromString(ctx context.Context, schemaString string) ( []client.CollectionDescription, - [][]client.IndexDescription, error, ) { source := source.NewSource(&source.Source{ @@ -39,7 +38,7 @@ func FromString(ctx context.Context, schemaString string) ( }, ) if err != nil { - return nil, nil, err + return nil, err } return fromAst(ctx, doc) @@ -48,23 +47,20 @@ func FromString(ctx context.Context, schemaString string) ( // fromAst parses a GQL AST into a set of collection descriptions. func fromAst(ctx context.Context, doc *ast.Document) ( []client.CollectionDescription, - [][]client.IndexDescription, error, ) { relationManager := NewRelationManager() descriptions := []client.CollectionDescription{} - indexes := [][]client.IndexDescription{} for _, def := range doc.Definitions { switch defType := def.(type) { case *ast.ObjectDefinition: - description, colIndexes, err := fromAstDefinition(ctx, relationManager, defType) + description, err := fromAstDefinition(ctx, relationManager, defType) if err != nil { - return nil, nil, err + return nil, err } descriptions = append(descriptions, description) - indexes = append(indexes, colIndexes) default: // Do nothing, ignore it and continue @@ -77,10 +73,10 @@ func fromAst(ctx context.Context, doc *ast.Document) ( // after all the collections have been processed. err := finalizeRelations(relationManager, descriptions) if err != nil { - return nil, nil, err + return nil, err } - return descriptions, indexes, nil + return descriptions, nil } // fromAstDefinition parses a AST object definition into a set of collection descriptions. @@ -88,7 +84,7 @@ func fromAstDefinition( ctx context.Context, relationManager *RelationManager, def *ast.ObjectDefinition, -) (client.CollectionDescription, []client.IndexDescription, error) { +) (client.CollectionDescription, error) { fieldDescriptions := []client.FieldDescription{ { Name: request.KeyFieldName, @@ -101,7 +97,7 @@ func fromAstDefinition( for _, field := range def.Fields { tmpFieldsDescriptions, err := fieldsFromAST(field, relationManager, def) if err != nil { - return client.CollectionDescription{}, nil, err + return client.CollectionDescription{}, err } fieldDescriptions = append(fieldDescriptions, tmpFieldsDescriptions...) @@ -110,7 +106,7 @@ func fromAstDefinition( if directive.Name.Value == "index" { index, err := fieldIndexFromAST(field, directive) if err != nil { - return client.CollectionDescription{}, nil, err + return client.CollectionDescription{}, err } indexDescriptions = append(indexDescriptions, index) } @@ -132,7 +128,7 @@ func fromAstDefinition( if directive.Name.Value == "index" { index, err := indexFromAST(directive) if err != nil { - return client.CollectionDescription{}, nil, err + return client.CollectionDescription{}, err } indexDescriptions = append(indexDescriptions, index) } @@ -144,7 +140,8 @@ func fromAstDefinition( Name: def.Name.Value, Fields: fieldDescriptions, }, - }, indexDescriptions, nil + Indexes: indexDescriptions, + }, nil } func isValidIndexName(name string) bool { diff --git a/request/graphql/schema/descriptions_test.go b/request/graphql/schema/descriptions_test.go index 0132d24291..2ce5e55dc9 100644 --- a/request/graphql/schema/descriptions_test.go +++ b/request/graphql/schema/descriptions_test.go @@ -58,6 +58,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, }, }, @@ -104,6 +105,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, { Name: "Author", @@ -132,6 +134,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, }, }, @@ -187,6 +190,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, { Name: "Author", @@ -224,6 +228,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, }, }, @@ -270,6 +275,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, { Name: "Author", @@ -298,6 +304,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, }, }, @@ -353,6 +360,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, { Name: "Author", @@ -390,6 +398,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, }, }, @@ -445,6 +454,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, { Name: "Author", @@ -482,6 +492,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, }, }, @@ -537,6 +548,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, { Name: "Author", @@ -568,6 +580,7 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, + Indexes: []client.IndexDescription{}, }, }, }, @@ -581,7 +594,7 @@ func TestSingleSimpleType(t *testing.T) { func runCreateDescriptionTest(t *testing.T, testcase descriptionTestCase) { ctx := context.Background() - descs, _, err := FromString(ctx, testcase.sdl) + descs, err := FromString(ctx, testcase.sdl) assert.NoError(t, err, testcase.description) assert.Equal(t, len(descs), len(testcase.targetDescs), testcase.description) diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index 3e53975da4..6337a7f4d8 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -340,12 +340,12 @@ func TestInvalidFieldIndex(t *testing.T) { func parseIndexAndTest(t *testing.T, testCase indexTestCase) { ctx := context.Background() - _, colIndexes, err := FromString(ctx, testCase.sdl) + cols, err := FromString(ctx, testCase.sdl) assert.NoError(t, err, testCase.description) - assert.Equal(t, len(colIndexes), 1, testCase.description) - assert.Equal(t, len(colIndexes[0]), len(testCase.targetDescriptions), testCase.description) + assert.Equal(t, len(cols), 1, testCase.description) + assert.Equal(t, len(cols[0].Indexes), len(testCase.targetDescriptions), testCase.description) - for i, d := range colIndexes[0] { + for i, d := range cols[0].Indexes { assert.Equal(t, testCase.targetDescriptions[i], d, testCase.description) } } @@ -353,7 +353,7 @@ func parseIndexAndTest(t *testing.T, testCase indexTestCase) { func parseInvalidIndexAndTest(t *testing.T, testCase invalidIndexTestCase) { ctx := context.Background() - _, _, err := FromString(ctx, testCase.sdl) + _, err := FromString(ctx, testCase.sdl) assert.EqualError(t, err, testCase.expectedErr, testCase.description) } From fa6d51cdac69e1cb0400df1b7ebefa9729027e21 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 May 2023 15:22:11 +0200 Subject: [PATCH 063/120] Add documentation to index description --- client/index.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/client/index.go b/client/index.go index 2608ae1466..4e7679313b 100644 --- a/client/index.go +++ b/client/index.go @@ -1,5 +1,6 @@ package client +// IndexDirection is the direction of an index. type IndexDirection string const ( @@ -7,19 +8,30 @@ const ( Descending IndexDirection = "DESC" ) +// IndexFieldDescription describes how a field is being indexed. type IndexedFieldDescription struct { Name string Direction IndexDirection } +// IndexDescription describes an index. type IndexDescription struct { - Name string - ID uint32 + // Name contains the name of the index. + Name string + // ID is the local identifier of this index. + ID uint32 + // Fields contains the fields that are being indexed. Fields []IndexedFieldDescription + // Unique indicates whether the index is unique. Unique bool } +// CollectionIndexDescription describes an index on a collection. +// It's useful for retrieving a list of indexes without having to +// retrieve the entire collection description. type CollectionIndexDescription struct { + // CollectionName contains the name of the collection. CollectionName string - Index IndexDescription + // Index contains the index description. + Index IndexDescription } From 836046a1bf80bab814d9ed7de0369168c78dba36 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 May 2023 16:08:27 +0200 Subject: [PATCH 064/120] Minor polish --- tests/bench/query/planner/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bench/query/planner/utils.go b/tests/bench/query/planner/utils.go index 5841cb460a..148347aa2f 100644 --- a/tests/bench/query/planner/utils.go +++ b/tests/bench/query/planner/utils.go @@ -107,7 +107,7 @@ func buildParser( return nil, err } - collectionDescriptions, _, err := gqlSchema.FromString(ctx, schema) + collectionDescriptions, err := gqlSchema.FromString(ctx, schema) if err != nil { return nil, err } From 02c0e60505a23c40a29a54514cda672a388ac3a0 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 May 2023 16:14:02 +0200 Subject: [PATCH 065/120] Remove accident files --- tests/integration/index/simple_test.go | 43 ----------- tests/integration/index/utils.go | 103 ------------------------- 2 files changed, 146 deletions(-) delete mode 100644 tests/integration/index/simple_test.go delete mode 100644 tests/integration/index/utils.go diff --git a/tests/integration/index/simple_test.go b/tests/integration/index/simple_test.go deleted file mode 100644 index 78219813aa..0000000000 --- a/tests/integration/index/simple_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package index - -import ( - "testing" - - testUtils "github.com/sourcenetwork/defradb/tests/integration" -) - -func TestSchemaWithIndexOnTheOnlyField(t *testing.T) { - test := testUtils.TestCase{ - Actions: []any{ - testUtils.SchemaUpdate{ - Schema: ` - type users { - name: String @index - } - `, - }, - createUserDocs(), - testUtils.Request{ - Request: ` - query @explain(type: execute) { - users(filter: {name: {_eq: "Shahzad"}}) { - name - } - }`, - Asserter: newExplainAsserter(2, 8, 1), - }, - }, - } - - testUtils.ExecuteTestCase(t, []string{"users"}, test) -} diff --git a/tests/integration/index/utils.go b/tests/integration/index/utils.go deleted file mode 100644 index ca4dc3981a..0000000000 --- a/tests/integration/index/utils.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package index - -import ( - "testing" - - testUtils "github.com/sourcenetwork/defradb/tests/integration" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type dataMap = map[string]any - -type explainResultAsserter struct { - iterations int - docFetches int - filterMatches int -} - -func (a explainResultAsserter) Assert(t *testing.T, result []dataMap) { - require.Len(t, result, 1, "Expected one result, got %d", len(result)) - explainNode, ok := result[0]["explain"].(dataMap) - require.True(t, ok, "Expected explain") - assert.Equal(t, explainNode["executionSuccess"], true) - assert.Equal(t, explainNode["sizeOfResult"], 1) - assert.Equal(t, explainNode["planExecutions"], uint64(2)) - selectTopNode, ok := explainNode["selectTopNode"].(dataMap) - require.True(t, ok, "Expected selectTopNode") - selectNode, ok := selectTopNode["selectNode"].(dataMap) - require.True(t, ok, "Expected selectNode") - scanNode, ok := selectNode["scanNode"].(dataMap) - require.True(t, ok, "Expected scanNode") - assert.Equal(t, scanNode["iterations"], uint64(a.iterations), - "Expected %d iterations, got %d", a.iterations, scanNode["iterations"]) - assert.Equal(t, scanNode["docFetches"], uint64(a.docFetches), - "Expected %d docFetches, got %d", a.docFetches, scanNode["docFetches"]) - assert.Equal(t, scanNode["filterMatches"], uint64(a.filterMatches), - "Expected %d filterMatches, got %d", a.filterMatches, scanNode["filterMatches"]) -} - -func newExplainAsserter(iterations, docFetched, filterMatcher int) *explainResultAsserter { - return &explainResultAsserter{ - iterations: iterations, - docFetches: docFetched, - filterMatches: filterMatcher, - } -} - -func createUserDocs() []testUtils.CreateDoc { - return []testUtils.CreateDoc{ - { - CollectionID: 0, - Doc: `{ - "name": "John" - }`, - }, - { - CollectionID: 0, - Doc: `{ - "name": "Islam" - }`, - }, - { - CollectionID: 0, - Doc: `{ - "name": "Andy" - }`, - }, - { - CollectionID: 0, - Doc: `{ - "name": "Shahzad" - }`, - }, - { - CollectionID: 0, - Doc: `{ - "name": "Fred" - }`, - }, - { - CollectionID: 0, - Doc: `{ - "name": "Orpheus" - }`, - }, - { - CollectionID: 0, - Doc: `{ - "name": "Addo" - }`, - }, - } -} From 05ae66a18e1a6610407b8fcd8ac4d775b7420650 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 May 2023 17:13:45 +0200 Subject: [PATCH 066/120] Remove unrelated changes --- tests/integration/test_case.go | 23 ++--------------------- tests/integration/utils2.go | 9 --------- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 0bb1bb0687..9a8f2300df 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -11,8 +11,6 @@ package tests import ( - "testing" - "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/config" @@ -153,20 +151,6 @@ type UpdateDoc struct { DontSync bool } -// ResultAsserter is an interface that can be implemented to provide custom result -// assertions. -type ResultAsserter interface { - // Assert will be called with the test and the result of the request. - Assert(t *testing.T, result []map[string]any) -} - -// ResultAsserterFunc is a function that can be used to implement the ResultAsserter -type ResultAsserterFunc func(*testing.T, []map[string]any) (bool, string) - -func (f ResultAsserterFunc) Assert(t *testing.T, result []map[string]any) { - f(t, result) -} - // Request represents a standard Defra (GQL) request. type Request struct { // NodeID may hold the ID (index) of a node to execute this request on. @@ -181,9 +165,6 @@ type Request struct { // The expected (data) results of the issued request. Results []map[string]any - // Asserter is an optional custom result asserter. - Asserter ResultAsserter - // Any error expected from the action. Optional. // // String can be a partial, and the test will pass if an error is returned that @@ -195,8 +176,8 @@ type Request struct { // // A new transaction will be created for the first TransactionRequest2 of any given // TransactionId. TransactionRequest2s will be submitted to the database in the order -// in which they are received (interleaving amongst other actions if provided), however -// they will not be committed until a TransactionCommit of matching TransactionId is +// in which they are recieved (interleaving amongst other actions if provided), however +// they will not be commited until a TransactionCommit of matching TransactionId is // provided. type TransactionRequest2 struct { // Used to identify the transaction for this to run against. diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index e86041687c..1f55bb439d 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1014,7 +1014,6 @@ func executeTransactionRequest( &result.GQL, action.Results, action.ExpectedError, - nil, // anyof is not yet supported by transactional requests 0, map[docFieldKey][]any{}, @@ -1073,7 +1072,6 @@ func executeRequest( &result.GQL, action.Results, action.ExpectedError, - action.Asserter, nodeID, anyOfByFieldKey, ) @@ -1144,7 +1142,6 @@ func executeSubscriptionRequest( finalResult, action.Results, action.ExpectedError, - nil, // anyof is not yet supported by subscription requests 0, map[docFieldKey][]any{}, @@ -1218,7 +1215,6 @@ func assertRequestResults( result *client.GQLResult, expectedResults []map[string]any, expectedError string, - asserter ResultAsserter, nodeID int, anyOfByField map[docFieldKey][]any, ) bool { @@ -1233,11 +1229,6 @@ func assertRequestResults( // Note: if result.Data == nil this panics (the panic seems useful while testing). resultantData := result.Data.([]map[string]any) - if asserter != nil { - asserter.Assert(t, resultantData) - return true - } - log.Info(ctx, "", logging.NewKV("RequestResults", result.Data)) // compare results From e12f57c054806524bd2ab678141836bb311efcce Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 May 2023 18:27:17 +0200 Subject: [PATCH 067/120] Add "mocks" folder to ignore list of codecov --- .github/codecov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/codecov.yml b/.github/codecov.yml index 60606a3007..cacd8f8180 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -62,5 +62,6 @@ comment: ignore: - "tests" + - "**/mocks/*" - "**/*_test.go" - "**/*.pb.go" From 8298eeb4eb1fea9a38534c2c964815722ef304bc Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 31 May 2023 12:15:11 +0200 Subject: [PATCH 068/120] Fix linter issues --- datastore/mocks/utils.go | 14 ++++++++++++-- db/collection_index.go | 5 +++-- db/fetcher/mocks/utils.go | 17 ++++++++++++++--- request/graphql/schema/collection.go | 24 ++++++++++++++++-------- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go index ded4bcb695..55c114ba8d 100644 --- a/datastore/mocks/utils.go +++ b/datastore/mocks/utils.go @@ -1,11 +1,21 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package mocks import ( "testing" ds "github.com/ipfs/go-datastore" - query "github.com/ipfs/go-datastore/query" - mock "github.com/stretchr/testify/mock" + "github.com/ipfs/go-datastore/query" + "github.com/stretchr/testify/mock" ) type MultiStoreTxn struct { diff --git a/db/collection_index.go b/db/collection_index.go index e9c2167788..38b4a24864 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -144,9 +144,10 @@ func (c *collection) collectIndexedFields() []*client.FieldDescription { fieldsMap := make(map[string]*client.FieldDescription) for _, index := range c.indexes { for _, field := range index.Description().Fields { - for _, colField := range c.desc.Schema.Fields { + for i := range c.desc.Schema.Fields { + colField := &c.desc.Schema.Fields[i] if field.Name == colField.Name { - fieldsMap[field.Name] = &colField + fieldsMap[field.Name] = colField break } } diff --git a/db/fetcher/mocks/utils.go b/db/fetcher/mocks/utils.go index e28b9ca1a8..8fa96ec7e1 100644 --- a/db/fetcher/mocks/utils.go +++ b/db/fetcher/mocks/utils.go @@ -1,11 +1,22 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package mocks import ( "testing" - client "github.com/sourcenetwork/defradb/client" - core "github.com/sourcenetwork/defradb/core" - mock "github.com/stretchr/testify/mock" + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/core" + + "github.com/stretchr/testify/mock" ) func NewStubbedFetcher(t *testing.T) *Fetcher { diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 4f774b41d0..17e5b89563 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -23,6 +23,14 @@ import ( "github.com/graphql-go/graphql/language/source" ) +const ( + indexDirectiveLabel = "index" + indexDirectivePropName = "name" + indexDirectivePropUnique = "unique" + indexDirectivePropFields = "fields" + indexDirectivePropDirections = "directions" +) + // FromString parses a GQL SDL string into a set of collection descriptions. func FromString(ctx context.Context, schemaString string) ( []client.CollectionDescription, @@ -103,7 +111,7 @@ func fromAstDefinition( fieldDescriptions = append(fieldDescriptions, tmpFieldsDescriptions...) for _, directive := range field.Directives { - if directive.Name.Value == "index" { + if directive.Name.Value == indexDirectiveLabel { index, err := fieldIndexFromAST(field, directive) if err != nil { return client.CollectionDescription{}, err @@ -125,7 +133,7 @@ func fromAstDefinition( }) for _, directive := range def.Directives { - if directive.Name.Value == "index" { + if directive.Name.Value == indexDirectiveLabel { index, err := indexFromAST(directive) if err != nil { return client.CollectionDescription{}, err @@ -171,7 +179,7 @@ func fieldIndexFromAST(field *ast.FieldDefinition, directive *ast.Directive) (cl } for _, arg := range directive.Arguments { switch arg.Name.Value { - case "name": + case indexDirectivePropName: nameVal, ok := arg.Value.(*ast.StringValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg @@ -180,7 +188,7 @@ func fieldIndexFromAST(field *ast.FieldDefinition, directive *ast.Directive) (cl if !isValidIndexName(desc.Name) { return client.IndexDescription{}, ErrIndexWithInvalidArg } - case "unique": + case indexDirectivePropUnique: boolVal, ok := arg.Value.(*ast.BooleanValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg @@ -198,7 +206,7 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { var directions *ast.ListValue for _, arg := range directive.Arguments { switch arg.Name.Value { - case "name": + case indexDirectivePropName: nameVal, ok := arg.Value.(*ast.StringValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg @@ -207,7 +215,7 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { if !isValidIndexName(desc.Name) { return client.IndexDescription{}, ErrIndexWithInvalidArg } - case "fields": + case indexDirectivePropFields: fieldsVal, ok := arg.Value.(*ast.ListValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg @@ -221,13 +229,13 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { Name: fieldVal.Value, }) } - case "directions": + case indexDirectivePropDirections: var ok bool directions, ok = arg.Value.(*ast.ListValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg } - case "unique": + case indexDirectivePropUnique: boolVal, ok := arg.Value.(*ast.BooleanValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg From 4598c17d9103110ecb0753318fa8e3eedf142a57 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 31 May 2023 12:23:04 +0200 Subject: [PATCH 069/120] Soft imports --- db/errors.go | 2 +- db/index.go | 1 + db/indexed_docs_test.go | 7 ++++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/db/errors.go b/db/errors.go index d809c54c87..5cc936ead9 100644 --- a/db/errors.go +++ b/db/errors.go @@ -157,7 +157,7 @@ func NewErrCanNotIndexInvalidFieldValue(inner error) error { return errors.Wrap(errCanNotIndexInvalidFieldValue, inner) } -// NewCanNotDeleteIndexedField returns a new error a failed attempt to delete an indexed field +// NewCanNotDeleteIndexedField returns a new error a failed attempt to delete an indexed field func NewCanNotDeleteIndexedField(inner error) error { return errors.Wrap(errCanNotDeleteIndexedField, inner) } diff --git a/db/index.go b/db/index.go index 4b2209cf25..5618f7d808 100644 --- a/db/index.go +++ b/db/index.go @@ -11,6 +11,7 @@ import ( ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index a45e9262b6..0b76b41f57 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -9,14 +9,15 @@ import ( "time" "github.com/ipfs/go-datastore/query" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore/mocks" "github.com/sourcenetwork/defradb/db/fetcher" fetcherMocks "github.com/sourcenetwork/defradb/db/fetcher/mocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" ) const testValuePrefix = "v" From 0f14e7df1654692bddf8e0012054efe20cba62a3 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 31 May 2023 12:30:04 +0200 Subject: [PATCH 070/120] Add copyright header --- db/index.go | 10 ++++++++++ db/indexed_docs_test.go | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/db/index.go b/db/index.go index 5618f7d808..208fcf3e1d 100644 --- a/db/index.go +++ b/db/index.go @@ -1,3 +1,13 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package db import ( diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 0b76b41f57..77735f91a1 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -1,3 +1,13 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package db import ( From 0e1f1d0207005cf04d12ef807abd7e3274a25a98 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 31 May 2023 13:16:03 +0200 Subject: [PATCH 071/120] Fix linter issues --- client/descriptions.go | 2 +- client/index.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/client/descriptions.go b/client/descriptions.go index fbba85fd35..7142bf3c70 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -29,7 +29,7 @@ type CollectionDescription struct { // Schema contains the data type information that this Collection uses. Schema SchemaDescription - + // Indexes contains the indexes that this Collection has. Indexes []IndexDescription } diff --git a/client/index.go b/client/index.go index 4e7679313b..d3e81a4cf1 100644 --- a/client/index.go +++ b/client/index.go @@ -1,3 +1,13 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package client // IndexDirection is the direction of an index. From f53ccabc92c870c15b1a14a28290c86e74c91854 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 31 May 2023 17:02:46 +0200 Subject: [PATCH 072/120] Add more tests and small fixes --- db/collection_index.go | 4 +-- db/index_test.go | 82 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index 38b4a24864..3548702858 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -70,7 +70,7 @@ func (db *db) getAllCollectionIndexes( indexes := make([]client.CollectionIndexDescription, 0) for res := range q.Next() { if res.Error != nil { - return nil, err + return nil, res.Error } var colDesk client.IndexDescription @@ -112,7 +112,7 @@ func (db *db) getCollectionIndexes( indexes := make([]client.IndexDescription, 0) for res := range q.Next() { if res.Error != nil { - return nil, err + return nil, res.Error } var colDesk client.IndexDescription diff --git a/db/index_test.go b/db/index_test.go index 26cb7b08cd..1c0cb0e060 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -563,6 +563,46 @@ func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { assert.ErrorIs(t, err, NewErrInvalidStoredIndexKey(key.String())) } +func TestGetIndexes_IfSystemStoreFails_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Unset() + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything). + Return(nil, errors.New("test error")) + + _, err := f.getAllIndexes() + assert.ErrorIs(t, err, NewErrFailedToCreateCollectionQuery(nil)) +} + +func TestGetIndexes_IfSystemStoreFails_ShouldCloseIterator(t *testing.T) { + f := newIndexTestFixture(t) + + mockedTxn := f.mockTxn() + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Unset() + q := mocks.NewQueryResultsWithValues(t) + q.EXPECT().Close().Return(nil) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Return(q, nil) + + _, _ = f.getAllIndexes() +} + +func TestGetIndexes_IfSystemStoreQueryIteratorFails_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Unset() + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}), nil) + + _, err := f.getAllIndexes() + assert.ErrorIs(t, err, testErr) +} + func TestGetIndexes_IfFailsToReadSeqNumber_ReturnError(t *testing.T) { testErr := errors.New("test error") @@ -631,14 +671,48 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) assert.Equal(t, productsIndexDesc, productIndexes[0]) } -func TestGetCollectionIndexes_IfStorageFails_ReturnError(t *testing.T) { +func TestGetCollectionIndexes_IfSystemStoreFails_ReturnError(t *testing.T) { f := newIndexTestFixture(t) - f.createUserCollectionIndexOnName() - f.db.Close(f.ctx) + mockedTxn := f.mockTxn() + mockedTxn.MockSystemstore = mocks.NewDSReaderWriter(t) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything). + Return(nil, errors.New("test error")) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore) _, err := f.getCollectionIndexes(usersColName) - assert.Error(t, err) + assert.ErrorIs(t, err, NewErrFailedToCreateCollectionQuery(nil)) +} + +func TestGetCollectionIndexes_IfSystemStoreFails_ShouldCloseIterator(t *testing.T) { + f := newIndexTestFixture(t) + + mockedTxn := f.mockTxn() + mockedTxn.MockSystemstore = mocks.NewDSReaderWriter(t) + query := mocks.NewQueryResultsWithValues(t) + query.EXPECT().Close().Return(nil) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Return(query, nil) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore) + + _, _ = f.getCollectionIndexes(usersColName) +} + +func TestGetCollectionIndexes_IfSystemStoreQueryIteratorFails_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn() + mockedTxn.MockSystemstore = mocks.NewDSReaderWriter(t) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}), nil) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore) + + _, err := f.getCollectionIndexes(usersColName) + assert.ErrorIs(t, err, testErr) } func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { From 2a5fc798832a8965646f994aa4973aa525700b8f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 1 Jun 2023 11:48:22 +0200 Subject: [PATCH 073/120] Add missing test --- db/indexed_docs_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 77735f91a1..a4e03fb80e 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -658,6 +658,34 @@ func TestNonUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { } } +func TestNonUniqueUpdate_IfFailsToReadIndexDescription_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + + err := doc.Set(usersNameFieldName, "Islam") + require.NoError(t, err) + + // retrieve the collection without index cached + usersCol, err := f.db.getCollectionByName(f.ctx, f.txn, usersColName) + require.NoError(t, err) + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn() + mockedTxn.MockSystemstore = mocks.NewDSReaderWriter(t) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Return(nil, testErr) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore) + mockedTxn.MockDatastore.EXPECT().Get(mock.Anything, mock.Anything).Unset() + mockedTxn.MockDatastore.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + + err = usersCol.WithTxn(mockedTxn).Update(f.ctx, doc) + require.ErrorIs(t, err, testErr) +} + func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { testError := errors.New("test error") From af2b96407ba6020d5c69eb58b2964eb6e6f6f46c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 1 Jun 2023 13:11:29 +0200 Subject: [PATCH 074/120] Polish --- db/index.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/db/index.go b/db/index.go index 208fcf3e1d..2c7ee82df6 100644 --- a/db/index.go +++ b/db/index.go @@ -77,10 +77,7 @@ func getFieldValConverter(kind client.FieldKind) func(any) ([]byte, error) { } case client.FieldKind_DATETIME: return func(val any) ([]byte, error) { - timeStrVal, ok := val.(string) - if !ok { - return nil, errors.New("invalid datetime value") - } + timeStrVal := val.(string) _, err := time.Parse(time.RFC3339, timeStrVal) if err != nil { return nil, err @@ -119,7 +116,7 @@ func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataS if err != nil { isNil = errors.Is(err, client.ErrFieldNotExist) if !isNil { - return core.IndexDataStoreKey{}, nil // @todo: test + return core.IndexDataStoreKey{}, nil } } From aeccb8c89d86909dbb154d8d12d952ee1fa19112 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 1 Jun 2023 13:40:31 +0200 Subject: [PATCH 075/120] Add missing tests --- db/index.go | 2 +- db/index_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/db/index.go b/db/index.go index 2c7ee82df6..87d172ef59 100644 --- a/db/index.go +++ b/db/index.go @@ -116,7 +116,7 @@ func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataS if err != nil { isNil = errors.Is(err, client.ErrFieldNotExist) if !isNil { - return core.IndexDataStoreKey{}, nil + return core.IndexDataStoreKey{}, nil } } diff --git a/db/index_test.go b/db/index_test.go index 1c0cb0e060..20e4a5b018 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -964,6 +964,39 @@ func TestDropIndex_IfFailsToQuerySystemStorage_ReturnError(t *testing.T) { require.ErrorIs(t, err, testErr) } +func TestDropIndex_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + testErr := errors.New("test error") + + mockedRootStore := mocks.NewRootStore(t) + mockedRootStore.EXPECT().NewTransaction(mock.Anything, mock.Anything).Return(nil, testErr) + f.db.rootstore = mockedRootStore + + err := f.users.DropIndex(f.ctx, testUsersColIndexName) + require.ErrorIs(t, err, testErr) +} + +func TestDropIndex_IfFailsToDeleteFromStorage_ShouldNotCache(t *testing.T) { + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn().ClearSystemStore() + systemStoreOn := mockedTxn.MockSystemstore.EXPECT() + systemStoreOn.Delete(mock.Anything, mock.Anything).Return(testErr) + f.stubSystemStore(systemStoreOn) + mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything).Maybe(). + Return(mocks.NewQueryResultsWithValues(t), nil) + + err := f.users.WithTxn(mockedTxn).DropIndex(f.ctx, testUsersColIndexName) + require.ErrorIs(t, err, testErr) +} + func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { f := newIndexTestFixture(t) _, err := f.createCollectionIndexFor(usersColName, client.IndexDescription{ From 8c770e11c48b48f3d5d292afd2c34ffb25935592 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 2 Jun 2023 10:31:00 +0200 Subject: [PATCH 076/120] Add more tests --- db/index.go | 40 ++++------- db/index_test.go | 175 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 167 insertions(+), 48 deletions(-) diff --git a/db/index.go b/db/index.go index 87d172ef59..9348f2641f 100644 --- a/db/index.go +++ b/db/index.go @@ -250,7 +250,12 @@ func (c *collection) CreateIndex( ctx context.Context, desc client.IndexDescription, ) (client.IndexDescription, error) { - index, err := c.createIndex(ctx, desc) + txn, err := c.getTxn(ctx, false) + if err != nil { + return client.IndexDescription{}, err + } + + index, err := c.createIndex(ctx, txn, desc) if err != nil { return client.IndexDescription{}, err } @@ -289,12 +294,8 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { return nil } -func (c *collection) dropAllIndexes(ctx context.Context) error { +func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) error { prefix := core.NewCollectionIndexKey(c.Name(), "") - txn, err := c.getTxn(ctx, false) - if err != nil { - return err - } q, err := txn.Systemstore().Query(ctx, query.Query{ Prefix: prefix.ToString(), }) @@ -352,13 +353,6 @@ func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]Colle } prefix := core.NewCollectionIndexKey(c.Name(), "") - if txn == nil { - var err error - txn, err = c.getTxn(ctx, true) - if err != nil { - return nil, err - } - } indexes, err := deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) if err != nil { return nil, err @@ -374,7 +368,11 @@ func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]Colle } func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { - indexes, err := c.getIndexes(ctx, nil) + txn, err := c.getTxn(ctx, true) + if err != nil { + return nil, err + } + indexes, err := c.getIndexes(ctx, txn) if err != nil { return nil, err } @@ -388,6 +386,7 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, func (c *collection) createIndex( ctx context.Context, + txn datastore.Txn, desc client.IndexDescription, ) (CollectionIndex, error) { err := validateIndexDescription(desc) @@ -400,12 +399,7 @@ func (c *collection) createIndex( return nil, err } - indexKey, err := c.processIndexName(ctx, &desc) - if err != nil { - return nil, err - } - - txn, err := c.getTxn(ctx, false) + indexKey, err := c.processIndexName(ctx, txn, &desc) if err != nil { return nil, err } @@ -456,13 +450,9 @@ func (c *collection) checkExistingFields( func (c *collection) processIndexName( ctx context.Context, + txn datastore.Txn, desc *client.IndexDescription, ) (core.CollectionIndexKey, error) { - txn, err := c.getTxn(ctx, true) - if err != nil { - return core.CollectionIndexKey{}, err - } - var indexKey core.CollectionIndexKey if desc.Name == "" { nameIncrement := 1 diff --git a/db/index_test.go b/db/index_test.go index 20e4a5b018..23d699d417 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -202,11 +202,6 @@ func (f *indexTestFixture) dropIndex(colName, indexName string) error { return f.db.dropCollectionIndex(f.ctx, f.txn, colName, indexName) } -func (f *indexTestFixture) dropAllIndexes(colName string) error { - col := (f.users.WithTxn(f.txn)).(*collection) - return col.dropAllIndexes(f.ctx) -} - func (f *indexTestFixture) countIndexPrefixes(colName, indexName string) int { prefix := core.NewCollectionIndexKey(usersColName, indexName) q, err := f.txn.Systemstore().Query(f.ctx, query.Query{ @@ -240,13 +235,7 @@ func (f *indexTestFixture) createCollectionIndexFor( collectionName string, desc client.IndexDescription, ) (client.IndexDescription, error) { - newDesc, err := f.db.createCollectionIndex(f.ctx, f.txn, collectionName, desc) - //if err != nil { - //return newDesc, err - //} - //f.txn, err = f.db.NewTxn(f.ctx, false) - //assert.NoError(f.t, err) - return newDesc, err + return f.db.createCollectionIndex(f.ctx, f.txn, collectionName, desc) } func (f *indexTestFixture) getAllIndexes() ([]client.CollectionIndexDescription, error) { @@ -434,18 +423,71 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { } func TestCreateIndex_IfStorageFails_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) + testErr := errors.New("test error") - name := "users_age_ASC" - desc := client.IndexDescription{ - Name: name, - Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, + testCases := []struct { + Name string + ExpectedError error + GetMockSystemstore func(t *testing.T) *mocks.DSReaderWriter + AlterDescription func(desc *client.IndexDescription) + }{ + { + Name: "call Has() for custom index name", + ExpectedError: testErr, + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Has(mock.Anything, mock.Anything).Unset() + store.EXPECT().Has(mock.Anything, mock.Anything).Return(false, testErr) + return store + }, + AlterDescription: func(desc *client.IndexDescription) {}, + }, + { + Name: "call Has() for generated index name", + ExpectedError: testErr, + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Has(mock.Anything, mock.Anything).Unset() + store.EXPECT().Has(mock.Anything, mock.Anything).Return(false, testErr) + return store + }, + AlterDescription: func(desc *client.IndexDescription) { + desc.Name = "" + }, + }, + { + Name: "fails to store index description", + ExpectedError: NewErrInvalidStoredIndex(nil), + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Unset() + key := core.NewCollectionIndexKey(usersColName, testUsersColIndexName) + store.EXPECT().Put(mock.Anything, key.ToDS(), mock.Anything).Return(testErr) + return store + }, + AlterDescription: func(desc *client.IndexDescription) {}, + }, } - f.db.Close(f.ctx) + for _, testCase := range testCases { + f := newIndexTestFixture(t) - _, err := f.createCollectionIndex(desc) - assert.Error(t, err) + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore = testCase.GetMockSystemstore(t) + f.stubSystemStore(mockedTxn.MockSystemstore.EXPECT()) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() + + desc := client.IndexDescription{ + Name: testUsersColIndexName, + Fields: []client.IndexedFieldDescription{{Name: usersNameFieldName}}, + } + testCase.AlterDescription(&desc) + + _, err := f.createCollectionIndex(desc) + assert.ErrorIs(t, err, testErr, testCase.Name) + } } func TestCreateIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { @@ -501,6 +543,19 @@ func TestCreateIndex_WithMultipleCollectionsAndIndexes_AssignIncrementedIDPerCol createIndexAndAssert(products, productsCategoryFieldName, 2) } +func TestCreateIndex_IfFailsToCreateTxn_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + testErr := errors.New("test error") + + mockedRootStore := mocks.NewRootStore(t) + mockedRootStore.EXPECT().NewTransaction(mock.Anything, mock.Anything).Return(nil, testErr) + f.db.rootstore = mockedRootStore + + _, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) + require.ErrorIs(t, err, testErr) +} + func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { f := newIndexTestFixture(t) @@ -964,7 +1019,7 @@ func TestDropIndex_IfFailsToQuerySystemStorage_ReturnError(t *testing.T) { require.ErrorIs(t, err, testErr) } -func TestDropIndex_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { +func TestDropIndex_IfFailsToCreateTxn_ReturnError(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -1015,7 +1070,7 @@ func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { assert.Equal(t, 2, f.countIndexPrefixes(usersColName, "")) - err = f.dropAllIndexes(usersColName) + err = f.users.dropAllIndexes(f.ctx, f.txn) assert.NoError(t, err) assert.Equal(t, 0, f.countIndexPrefixes(usersColName, "")) @@ -1027,6 +1082,80 @@ func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { f.db.Close(f.ctx) - err := f.dropAllIndexes(usersColName) + err := f.users.dropAllIndexes(f.ctx, f.txn) assert.Error(t, err) } + +func TestDropAllIndexes_IfSystemStorageFails_ReturnError(t *testing.T) { + testErr := errors.New("test error") + + testCases := []struct { + Name string + ExpectedError error + GetMockSystemstore func(t *testing.T) *mocks.DSReaderWriter + }{ + { + Name: "Query fails", + ExpectedError: testErr, + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Query(mock.Anything, mock.Anything).Unset() + store.EXPECT().Query(mock.Anything, mock.Anything).Return(nil, testErr) + return store + }, + }, + { + Name: "Query iterator fails", + ExpectedError: testErr, + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}), nil) + return store + }, + }, + { + Name: "System storage fails to delete", + ExpectedError: NewErrInvalidStoredIndex(nil), + GetMockSystemstore: func(t *testing.T) *mocks.DSReaderWriter { + store := mocks.NewDSReaderWriter(t) + store.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithValues(t, []byte{}), nil) + store.EXPECT().Delete(mock.Anything, mock.Anything).Maybe().Return(testErr) + return store + }, + }, + } + + for _, testCase := range testCases { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore = testCase.GetMockSystemstore(t) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() + + err := f.users.dropAllIndexes(f.ctx, f.txn) + assert.ErrorIs(t, err, testErr, testCase.Name) + } +} + +func TestDropAllIndexes_ShouldCloseQueryIterator(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore = mocks.NewDSReaderWriter(t) + q := mocks.NewQueryResultsWithValues(t, []byte{}) + q.EXPECT().Close().Unset() + q.EXPECT().Close().Return(nil) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Return(q, nil) + mockedTxn.MockSystemstore.EXPECT().Delete(mock.Anything, mock.Anything).Maybe().Return(nil) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() + + _ = f.users.dropAllIndexes(f.ctx, f.txn) +} From 14f01cfec815bbd742a614eb3714ec173adc55fb Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 2 Jun 2023 15:07:28 +0200 Subject: [PATCH 077/120] Small refactoring --- db/index.go | 61 +++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/db/index.go b/db/index.go index 9348f2641f..3c9a825431 100644 --- a/db/index.go +++ b/db/index.go @@ -170,13 +170,13 @@ func (i *collectionSimpleIndex) Update( return i.Save(ctx, txn, newDoc) } -func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn) error { - prefixKey := core.IndexDataStoreKey{} - prefixKey.CollectionID = strconv.Itoa(int(i.collection.ID())) - prefixKey.IndexID = strconv.Itoa(int(i.desc.ID)) - q, err := txn.Datastore().Query(ctx, query.Query{ - Prefix: prefixKey.ToString(), - }) +func iteratePrefixKeys( + ctx context.Context, + prefix string, + storage ds.Read, + execFunc func(context.Context, ds.Key) error, +) error { + q, err := storage.Query(ctx, query.Query{Prefix: prefix}) if err != nil { return err } @@ -190,14 +190,30 @@ func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn if res.Error != nil { return res.Error } - err = txn.Datastore().Delete(ctx, ds.NewKey(res.Key)) + err = execFunc(ctx, ds.NewKey(res.Key)) if err != nil { - return NewCanNotDeleteIndexedField(err) + return err } } return nil } +func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn) error { + prefixKey := core.IndexDataStoreKey{} + prefixKey.CollectionID = strconv.Itoa(int(i.collection.ID())) + prefixKey.IndexID = strconv.Itoa(int(i.desc.ID)) + + err := iteratePrefixKeys(ctx, prefixKey.ToString(), txn.Datastore(), + func(ctx context.Context, key ds.Key) error { + err := txn.Datastore().Delete(ctx, key) + if err != nil { + return NewCanNotDeleteIndexedField(err) + } + return nil + }) + + return err +} func (i *collectionSimpleIndex) Name() string { return i.desc.Name @@ -296,28 +312,13 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) error { prefix := core.NewCollectionIndexKey(c.Name(), "") - q, err := txn.Systemstore().Query(ctx, query.Query{ - Prefix: prefix.ToString(), - }) - if err != nil { - return err - } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() - for res := range q.Next() { - if res.Error != nil { - return res.Error - } - err = txn.Systemstore().Delete(ctx, ds.NewKey(res.Key)) - if err != nil { - return err - } - } - return nil + err := iteratePrefixKeys(ctx, prefix.ToString(), txn.Systemstore(), + func(ctx context.Context, key ds.Key) error { + return txn.Systemstore().Delete(ctx, key) + }) + + return err } func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Read) ([]T, error) { From 5466669511e526664c37193ad013b6c885393b59 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 2 Jun 2023 16:19:16 +0200 Subject: [PATCH 078/120] Add readme for tests --- tests/README.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..7ff95efb3e --- /dev/null +++ b/tests/README.md @@ -0,0 +1,37 @@ +# Tests + +This directory contains two types of tests: benchmark tests (located in the bench directory) and integration tests (located in the integration directory). +In addition to these, unit tests are also distributed among the other directories of the source code. + +## Test Types + +### Benchmark Tests + +The bench directory contains benchmark tests that are used to measure and monitor the performance of the database. + +### Integration Tests + +The integration directory contains integration tests that ensure different components of the system work together correctly. + +### Unit Tests + +Unit tests are spread throughout the source code and are located in the same directories as the code they are testing. +These tests focus on small, isolated parts of the code to ensure each part is working as expected. + +## Mocks + +For unit tests, we sometimes use mocks. Mocks are automatically generated from Go interfaces using the mockery tool. +This helps to isolate the code being tested and provide more focused and reliable tests. + +The mocks are generated into a separate mocks directory. You can generate a mock for a specific interface using the following command: + +```shell +mockery --name --with-expecter +``` + +Here, `--name` specifies the name of the interface for which to generate the mock. + +The `--with-expecter` option adds a helper struct for each method, making the mock strongly typed. +This leads to more generated code, but it removes the need to pass strings around and increases type safety. + +For more information on mockery, please refer to the [official repository](https://github.com/vektra/mockery). From b869790ec70ea82fd9159509bf1dded4aa046d76 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 2 Jun 2023 18:15:12 +0200 Subject: [PATCH 079/120] Make struct private for db package --- client/descriptions.go | 2 +- client/index.go | 10 ---------- db/collection_index.go | 16 +++++++++++++--- db/index_test.go | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/client/descriptions.go b/client/descriptions.go index 7142bf3c70..8570cb6201 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -30,7 +30,7 @@ type CollectionDescription struct { // Schema contains the data type information that this Collection uses. Schema SchemaDescription - // Indexes contains the indexes that this Collection has. + // Indexes contains the secondary indexes that this Collection has. Indexes []IndexDescription } diff --git a/client/index.go b/client/index.go index d3e81a4cf1..0e8909850c 100644 --- a/client/index.go +++ b/client/index.go @@ -35,13 +35,3 @@ type IndexDescription struct { // Unique indicates whether the index is unique. Unique bool } - -// CollectionIndexDescription describes an index on a collection. -// It's useful for retrieving a list of indexes without having to -// retrieve the entire collection description. -type CollectionIndexDescription struct { - // CollectionName contains the name of the collection. - CollectionName string - // Index contains the index description. - Index IndexDescription -} diff --git a/db/collection_index.go b/db/collection_index.go index 3548702858..98c70c3c1b 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -21,6 +21,16 @@ import ( "github.com/sourcenetwork/defradb/datastore" ) +// collectionIndexDescription describes an index on a collection. +// It's useful for retrieving a list of indexes without having to +// retrieve the entire collection description. +type collectionIndexDescription struct { + // CollectionName contains the name of the collection. + CollectionName string + // Index contains the index description. + Index client.IndexDescription +} + // createCollectionIndex creates a new collection index and saves it to the database in its system store. func (db *db) createCollectionIndex( ctx context.Context, @@ -53,7 +63,7 @@ func (db *db) dropCollectionIndex( func (db *db) getAllCollectionIndexes( ctx context.Context, txn datastore.Txn, -) ([]client.CollectionIndexDescription, error) { +) ([]collectionIndexDescription, error) { prefix := core.NewCollectionIndexKey("", "") q, err := txn.Systemstore().Query(ctx, query.Query{ Prefix: prefix.ToString(), @@ -67,7 +77,7 @@ func (db *db) getAllCollectionIndexes( } }() - indexes := make([]client.CollectionIndexDescription, 0) + indexes := make([]collectionIndexDescription, 0) for res := range q.Next() { if res.Error != nil { return nil, res.Error @@ -82,7 +92,7 @@ func (db *db) getAllCollectionIndexes( if err != nil { return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) } - indexes = append(indexes, client.CollectionIndexDescription{ + indexes = append(indexes, collectionIndexDescription{ CollectionName: indexKey.CollectionName, Index: colDesk, }) diff --git a/db/index_test.go b/db/index_test.go index 23d699d417..f1a6d9cb48 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -238,7 +238,7 @@ func (f *indexTestFixture) createCollectionIndexFor( return f.db.createCollectionIndex(f.ctx, f.txn, collectionName, desc) } -func (f *indexTestFixture) getAllIndexes() ([]client.CollectionIndexDescription, error) { +func (f *indexTestFixture) getAllIndexes() ([]collectionIndexDescription, error) { return f.db.getAllCollectionIndexes(f.ctx, f.txn) } From a0c759b29174fb8cfb2ce82b4f196453ea0d35b6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 2 Jun 2023 18:21:56 +0200 Subject: [PATCH 080/120] Remove unique --- client/index.go | 2 - db/index_test.go | 1 - request/graphql/schema/collection.go | 13 ------ request/graphql/schema/index_test.go | 69 +--------------------------- 4 files changed, 1 insertion(+), 84 deletions(-) diff --git a/client/index.go b/client/index.go index 0e8909850c..3fa829b284 100644 --- a/client/index.go +++ b/client/index.go @@ -32,6 +32,4 @@ type IndexDescription struct { ID uint32 // Fields contains the fields that are being indexed. Fields []IndexedFieldDescription - // Unique indicates whether the index is unique. - Unique bool } diff --git a/db/index_test.go b/db/index_test.go index f1a6d9cb48..d9004a5d13 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -296,7 +296,6 @@ func TestCreateIndex_IfValidInput_CreateIndex(t *testing.T) { assert.NoError(t, err) assert.Equal(t, desc.Name, resultDesc.Name) assert.Equal(t, desc.Fields, resultDesc.Fields) - assert.Equal(t, desc.Unique, resultDesc.Unique) } func TestCreateIndex_IfFieldNameIsEmpty_ReturnError(t *testing.T) { diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 17e5b89563..51dee05f85 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -26,7 +26,6 @@ import ( const ( indexDirectiveLabel = "index" indexDirectivePropName = "name" - indexDirectivePropUnique = "unique" indexDirectivePropFields = "fields" indexDirectivePropDirections = "directions" ) @@ -188,12 +187,6 @@ func fieldIndexFromAST(field *ast.FieldDefinition, directive *ast.Directive) (cl if !isValidIndexName(desc.Name) { return client.IndexDescription{}, ErrIndexWithInvalidArg } - case indexDirectivePropUnique: - boolVal, ok := arg.Value.(*ast.BooleanValue) - if !ok { - return client.IndexDescription{}, ErrIndexWithInvalidArg - } - desc.Unique = boolVal.Value default: return client.IndexDescription{}, ErrIndexWithUnknownArg } @@ -235,12 +228,6 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg } - case indexDirectivePropUnique: - boolVal, ok := arg.Value.(*ast.BooleanValue) - if !ok { - return client.IndexDescription{}, ErrIndexWithInvalidArg - } - desc.Unique = boolVal.Value default: return client.IndexDescription{}, ErrIndexWithUnknownArg } diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index 6337a7f4d8..bd5d3b090d 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -30,7 +30,6 @@ func TestStructIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - Unique: false, }, }, }, @@ -46,30 +45,6 @@ func TestStructIndex(t *testing.T) { }, }, }, - { - description: "Unique index", - sdl: `type user @index(fields: ["name"], unique: true) {}`, - targetDescriptions: []client.IndexDescription{ - { - Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, - }, - Unique: true, - }, - }, - }, - { - description: "Index explicitly not unique", - sdl: `type user @index(fields: ["name"], unique: false) {}`, - targetDescriptions: []client.IndexDescription{ - { - Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, - }, - Unique: false, - }, - }, - }, { description: "Index with explicit ascending field", sdl: `type user @index(fields: ["name"], directions: [ASC]) {}`, @@ -125,7 +100,7 @@ func TestInvalidStructIndex(t *testing.T) { cases := []invalidIndexTestCase{ { description: "missing 'fields' argument", - sdl: `type user @index(name: "userIndex", unique: true) {}`, + sdl: `type user @index(name: "userIndex") {}`, expectedErr: errIndexMissingFields, }, { @@ -158,11 +133,6 @@ func TestInvalidStructIndex(t *testing.T) { sdl: `type user @index(name: "user!name", fields: ["name"]) {}`, expectedErr: errIndexInvalidArgument, }, - { - description: "invalid 'unique' value type", - sdl: `type user @index(fields: ["name"], unique: "true") {}`, - expectedErr: errIndexInvalidArgument, - }, { description: "invalid 'fields' value type (not a list)", sdl: `type user @index(fields: "name") {}`, @@ -218,7 +188,6 @@ func TestFieldIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - Unique: false, }, }, }, @@ -233,35 +202,6 @@ func TestFieldIndex(t *testing.T) { Fields: []client.IndexedFieldDescription{ {Name: "name", Direction: client.Ascending}, }, - Unique: false, - }, - }, - }, - { - description: "unique field index", - sdl: `type user { - name: String @index(unique: true) - }`, - targetDescriptions: []client.IndexDescription{ - { - Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, - }, - Unique: true, - }, - }, - }, - { - description: "field index explicitly not unique", - sdl: `type user { - name: String @index(unique: false) - }`, - targetDescriptions: []client.IndexDescription{ - { - Fields: []client.IndexedFieldDescription{ - {Name: "name", Direction: client.Ascending}, - }, - Unique: false, }, }, }, @@ -323,13 +263,6 @@ func TestInvalidFieldIndex(t *testing.T) { }`, expectedErr: errIndexInvalidArgument, }, - { - description: "invalid 'unique' value type", - sdl: `type user { - name: String @index(unique: "true") - }`, - expectedErr: errIndexInvalidArgument, - }, } for _, test := range cases { From 29ce194f0a148ddb94fdc5914675eee6960efd1d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 5 Jun 2023 12:28:10 +0200 Subject: [PATCH 081/120] Index existing docs on index creation --- db/collection_get.go | 8 +- db/collection_index.go | 327 ++++++++++++++++++++++++++++++++++++++++ db/index.go | 262 -------------------------------- db/indexed_docs_test.go | 116 +++++++++++++- 4 files changed, 443 insertions(+), 270 deletions(-) diff --git a/db/collection_get.go b/db/collection_get.go index 7225d1215a..836842d094 100644 --- a/db/collection_get.go +++ b/db/collection_get.go @@ -17,7 +17,6 @@ import ( "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/db/base" - "github.com/sourcenetwork/defradb/db/fetcher" ) func (c *collection) Get(ctx context.Context, key client.DocKey, showDeleted bool) (*client.Document, error) { @@ -52,12 +51,7 @@ func (c *collection) get( showDeleted bool, ) (*client.Document, error) { // create a new document fetcher - var df fetcher.Fetcher - if c.fetcherFactory != nil { - df = c.fetcherFactory() - } else { - df = new(fetcher.DocumentFetcher) - } + df := c.newFetcher() desc := &c.desc // initialize it with the primary index err := df.Init(&c.desc, fields, false, showDeleted) diff --git a/db/collection_index.go b/db/collection_index.go index 98c70c3c1b..e21b9752db 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -13,12 +13,18 @@ package db import ( "context" "encoding/json" + "fmt" + "strconv" + "strings" + ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/db/base" + "github.com/sourcenetwork/defradb/db/fetcher" ) // collectionIndexDescription describes an index on a collection. @@ -191,3 +197,324 @@ func (c *collection) updateIndex( } return nil } + +func (c *collection) CreateIndex( + ctx context.Context, + desc client.IndexDescription, +) (client.IndexDescription, error) { + txn, err := c.getTxn(ctx, false) + if err != nil { + return client.IndexDescription{}, err + } + + index, err := c.createIndex(ctx, txn, desc) + if err != nil { + return client.IndexDescription{}, err + } + if c.isIndexCached { + c.indexes = append(c.indexes, index) + } + err = c.indexExistingDocs(ctx, txn, index) + if err != nil { + return client.IndexDescription{}, err + } + return index.Description(), nil +} + +func (c *collection) newFetcher() fetcher.Fetcher { + if c.fetcherFactory != nil { + return c.fetcherFactory() + } else { + return new(fetcher.DocumentFetcher) + } +} +func (c *collection) indexExistingDocs( + ctx context.Context, + txn datastore.Txn, + index CollectionIndex, +) error { + df := c.newFetcher() + + fields := make([]*client.FieldDescription, 0, 1) + for _, field := range index.Description().Fields { + for i := range c.desc.Schema.Fields { + colField := &c.desc.Schema.Fields[i] + if field.Name == colField.Name { + fields = append(fields, colField) + break + } + } + } + + err := df.Init(&c.desc, fields, false, false) + if err != nil { + _ = df.Close() + return err + } + start := base.MakeCollectionKey(c.desc) + spans := core.NewSpans(core.NewSpan(start, start.PrefixEnd())) + + err = df.Start(ctx, txn, spans) + if err != nil { + _ = df.Close() + return err + } + + var doc *client.Document + for { + doc, err = df.FetchNextDecoded(ctx) + if err != nil { + _ = df.Close() + return err + } + if doc == nil { + break + } + err = index.Save(ctx, txn, doc) + if err != nil { + return err + } + } + + return df.Close() +} + +func (c *collection) DropIndex(ctx context.Context, indexName string) error { + key := core.NewCollectionIndexKey(c.Name(), indexName) + + txn, err := c.getTxn(ctx, false) + if err != nil { + return err + } + _, err = c.getIndexes(ctx, txn) + if err != nil { + return err + } + for i := range c.indexes { + if c.indexes[i].Name() == indexName { + err = c.indexes[i].RemoveAll(ctx, txn) + if err != nil { + return err + } + c.indexes = append(c.indexes[:i], c.indexes[i+1:]...) + break + } + } + err = txn.Systemstore().Delete(ctx, key.ToDS()) + if err != nil { + return err + } + + return nil +} + +func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) error { + prefix := core.NewCollectionIndexKey(c.Name(), "") + + err := iteratePrefixKeys(ctx, prefix.ToString(), txn.Systemstore(), + func(ctx context.Context, key ds.Key) error { + return txn.Systemstore().Delete(ctx, key) + }) + + return err +} + +func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]CollectionIndex, error) { + if c.isIndexCached { + return c.indexes, nil + } + + prefix := core.NewCollectionIndexKey(c.Name(), "") + indexes, err := deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) + if err != nil { + return nil, err + } + colIndexes := make([]CollectionIndex, 0, len(indexes)) + for _, index := range indexes { + colIndexes = append(colIndexes, NewCollectionIndex(c, index)) + } + + c.indexes = colIndexes + c.isIndexCached = true + return colIndexes, nil +} + +func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { + txn, err := c.getTxn(ctx, true) + if err != nil { + return nil, err + } + indexes, err := c.getIndexes(ctx, txn) + if err != nil { + return nil, err + } + indexDescriptions := make([]client.IndexDescription, 0, len(indexes)) + for _, index := range indexes { + indexDescriptions = append(indexDescriptions, index.Description()) + } + + return indexDescriptions, nil +} + +func (c *collection) createIndex( + ctx context.Context, + txn datastore.Txn, + desc client.IndexDescription, +) (CollectionIndex, error) { + err := validateIndexDescription(desc) + if err != nil { + return nil, err + } + + err = c.checkExistingFields(ctx, desc.Fields) + if err != nil { + return nil, err + } + + indexKey, err := c.processIndexName(ctx, txn, &desc) + if err != nil { + return nil, err + } + + colSeq, err := c.db.getSequence(ctx, txn, fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, c.ID())) + if err != nil { + return nil, err + } + colID, err := colSeq.next(ctx, txn) + if err != nil { + return nil, err + } + desc.ID = uint32(colID) + + buf, err := json.Marshal(desc) + if err != nil { + return nil, err + } + + err = txn.Systemstore().Put(ctx, indexKey.ToDS(), buf) + if err != nil { + return nil, err + } + colIndex := NewCollectionIndex(c, desc) + return colIndex, nil +} + +func (c *collection) checkExistingFields( + ctx context.Context, + fields []client.IndexedFieldDescription, +) error { + collectionFields := c.Description().Schema.Fields + for _, field := range fields { + found := false + fieldLower := strings.ToLower(field.Name) + for _, colField := range collectionFields { + if fieldLower == strings.ToLower(colField.Name) { + found = true + break + } + } + if !found { + return NewErrNonExistingFieldForIndex(field.Name) + } + } + return nil +} + +func (c *collection) processIndexName( + ctx context.Context, + txn datastore.Txn, + desc *client.IndexDescription, +) (core.CollectionIndexKey, error) { + var indexKey core.CollectionIndexKey + if desc.Name == "" { + nameIncrement := 1 + for { + desc.Name = generateIndexName(c, desc.Fields, nameIncrement) + indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) + exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) + if err != nil { + return core.CollectionIndexKey{}, err + } + if !exists { + break + } + nameIncrement++ + } + } else { + indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) + exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) + if err != nil { + return core.CollectionIndexKey{}, err + } + if exists { + return core.CollectionIndexKey{}, ErrIndexWithNameAlreadyExists + } + } + return indexKey, nil +} + +func validateIndexDescription(desc client.IndexDescription) error { + if desc.ID != 0 { + return NewErrNonZeroIndexIDProvided(desc.ID) + } + if len(desc.Fields) == 0 { + return ErrIndexMissingFields + } + if len(desc.Fields) == 1 && desc.Fields[0].Direction == client.Descending { + return ErrIndexSingleFieldWrongDirection + } + for i := range desc.Fields { + if desc.Fields[i].Name == "" { + return ErrIndexFieldMissingName + } + if desc.Fields[i].Direction == "" { + desc.Fields[i].Direction = client.Ascending + } + } + return nil +} + +func generateIndexName(col client.Collection, fields []client.IndexedFieldDescription, inc int) string { + sb := strings.Builder{} + direction := "ASC" + //if fields[0].Direction == client.Descending { + //direction = "DESC" + //} + sb.WriteString(col.Name()) + sb.WriteByte('_') + sb.WriteString(fields[0].Name) + sb.WriteByte('_') + sb.WriteString(direction) + if inc > 1 { + sb.WriteByte('_') + sb.WriteString(strconv.Itoa(inc)) + } + return sb.String() +} + +func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Read) ([]T, error) { + q, err := storage.Query(ctx, query.Query{Prefix: prefix}) + if err != nil { + return nil, NewErrFailedToCreateCollectionQuery(err) + } + defer func() { + if err := q.Close(); err != nil { + log.ErrorE(ctx, "Failed to close collection query", err) + } + }() + + elements := make([]T, 0) + for res := range q.Next() { + if res.Error != nil { + return nil, res.Error + } + + var element T + err = json.Unmarshal(res.Value, &element) + if err != nil { + return nil, NewErrInvalidStoredIndex(err) + } + elements = append(elements, element) + } + return elements, nil +} diff --git a/db/index.go b/db/index.go index 3c9a825431..f14f9a4ea2 100644 --- a/db/index.go +++ b/db/index.go @@ -12,10 +12,7 @@ package db import ( "context" - "encoding/json" - "fmt" "strconv" - "strings" "time" ds "github.com/ipfs/go-datastore" @@ -222,262 +219,3 @@ func (i *collectionSimpleIndex) Name() string { func (i *collectionSimpleIndex) Description() client.IndexDescription { return i.desc } - -func validateIndexDescription(desc client.IndexDescription) error { - if desc.ID != 0 { - return NewErrNonZeroIndexIDProvided(desc.ID) - } - if len(desc.Fields) == 0 { - return ErrIndexMissingFields - } - if len(desc.Fields) == 1 && desc.Fields[0].Direction == client.Descending { - return ErrIndexSingleFieldWrongDirection - } - for i := range desc.Fields { - if desc.Fields[i].Name == "" { - return ErrIndexFieldMissingName - } - if desc.Fields[i].Direction == "" { - desc.Fields[i].Direction = client.Ascending - } - } - return nil -} - -func generateIndexName(col client.Collection, fields []client.IndexedFieldDescription, inc int) string { - sb := strings.Builder{} - direction := "ASC" - //if fields[0].Direction == client.Descending { - //direction = "DESC" - //} - sb.WriteString(col.Name()) - sb.WriteByte('_') - sb.WriteString(fields[0].Name) - sb.WriteByte('_') - sb.WriteString(direction) - if inc > 1 { - sb.WriteByte('_') - sb.WriteString(strconv.Itoa(inc)) - } - return sb.String() -} - -func (c *collection) CreateIndex( - ctx context.Context, - desc client.IndexDescription, -) (client.IndexDescription, error) { - txn, err := c.getTxn(ctx, false) - if err != nil { - return client.IndexDescription{}, err - } - - index, err := c.createIndex(ctx, txn, desc) - if err != nil { - return client.IndexDescription{}, err - } - if c.isIndexCached { - c.indexes = append(c.indexes, index) - } - return index.Description(), nil -} - -func (c *collection) DropIndex(ctx context.Context, indexName string) error { - key := core.NewCollectionIndexKey(c.Name(), indexName) - - txn, err := c.getTxn(ctx, false) - if err != nil { - return err - } - _, err = c.getIndexes(ctx, txn) - if err != nil { - return err - } - for i := range c.indexes { - if c.indexes[i].Name() == indexName { - err = c.indexes[i].RemoveAll(ctx, txn) - if err != nil { - return err - } - c.indexes = append(c.indexes[:i], c.indexes[i+1:]...) - break - } - } - err = txn.Systemstore().Delete(ctx, key.ToDS()) - if err != nil { - return err - } - - return nil -} - -func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) error { - prefix := core.NewCollectionIndexKey(c.Name(), "") - - err := iteratePrefixKeys(ctx, prefix.ToString(), txn.Systemstore(), - func(ctx context.Context, key ds.Key) error { - return txn.Systemstore().Delete(ctx, key) - }) - - return err -} - -func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Read) ([]T, error) { - q, err := storage.Query(ctx, query.Query{Prefix: prefix}) - if err != nil { - return nil, NewErrFailedToCreateCollectionQuery(err) - } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() - - elements := make([]T, 0) - for res := range q.Next() { - if res.Error != nil { - return nil, res.Error - } - - var element T - err = json.Unmarshal(res.Value, &element) - if err != nil { - return nil, NewErrInvalidStoredIndex(err) - } - elements = append(elements, element) - } - return elements, nil -} - -func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]CollectionIndex, error) { - if c.isIndexCached { - return c.indexes, nil - } - - prefix := core.NewCollectionIndexKey(c.Name(), "") - indexes, err := deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) - if err != nil { - return nil, err - } - colIndexes := make([]CollectionIndex, 0, len(indexes)) - for _, index := range indexes { - colIndexes = append(colIndexes, NewCollectionIndex(c, index)) - } - - c.indexes = colIndexes - c.isIndexCached = true - return colIndexes, nil -} - -func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { - txn, err := c.getTxn(ctx, true) - if err != nil { - return nil, err - } - indexes, err := c.getIndexes(ctx, txn) - if err != nil { - return nil, err - } - indexDescriptions := make([]client.IndexDescription, 0, len(indexes)) - for _, index := range indexes { - indexDescriptions = append(indexDescriptions, index.Description()) - } - - return indexDescriptions, nil -} - -func (c *collection) createIndex( - ctx context.Context, - txn datastore.Txn, - desc client.IndexDescription, -) (CollectionIndex, error) { - err := validateIndexDescription(desc) - if err != nil { - return nil, err - } - - err = c.checkExistingFields(ctx, desc.Fields) - if err != nil { - return nil, err - } - - indexKey, err := c.processIndexName(ctx, txn, &desc) - if err != nil { - return nil, err - } - - colSeq, err := c.db.getSequence(ctx, txn, fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, c.ID())) - if err != nil { - return nil, err - } - colID, err := colSeq.next(ctx, txn) - if err != nil { - return nil, err - } - desc.ID = uint32(colID) - - buf, err := json.Marshal(desc) - if err != nil { - return nil, err - } - - err = txn.Systemstore().Put(ctx, indexKey.ToDS(), buf) - if err != nil { - return nil, err - } - colIndex := NewCollectionIndex(c, desc) - return colIndex, nil -} - -func (c *collection) checkExistingFields( - ctx context.Context, - fields []client.IndexedFieldDescription, -) error { - collectionFields := c.Description().Schema.Fields - for _, field := range fields { - found := false - fieldLower := strings.ToLower(field.Name) - for _, colField := range collectionFields { - if fieldLower == strings.ToLower(colField.Name) { - found = true - break - } - } - if !found { - return NewErrNonExistingFieldForIndex(field.Name) - } - } - return nil -} - -func (c *collection) processIndexName( - ctx context.Context, - txn datastore.Txn, - desc *client.IndexDescription, -) (core.CollectionIndexKey, error) { - var indexKey core.CollectionIndexKey - if desc.Name == "" { - nameIncrement := 1 - for { - desc.Name = generateIndexName(c, desc.Fields, nameIncrement) - indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) - exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) - if err != nil { - return core.CollectionIndexKey{}, err - } - if !exists { - break - } - nameIncrement++ - } - } else { - indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) - exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) - if err != nil { - return core.CollectionIndexKey{}, err - } - if exists { - return core.CollectionIndexKey{}, ErrIndexWithNameAlreadyExists - } - } - return indexKey, nil -} diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index a4e03fb80e..45276f456c 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -252,7 +252,6 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveDocToCollection(doc, f.users) - //f.commitTxn() key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() @@ -507,6 +506,121 @@ func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { assert.Len(t, data, 0) } +func TestNonUniqueCreate_ShouldIndexExistingDocs(t *testing.T) { + f := newIndexTestFixture(t) + + doc1 := f.newUserDoc("John", 21) + f.saveDocToCollection(doc1, f.users) + doc2 := f.newUserDoc("Islam", 18) + f.saveDocToCollection(doc2, f.users) + + f.createUserCollectionIndexOnName() + + key1 := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc1).Build() + key2 := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc2).Build() + + data, err := f.txn.Datastore().Get(f.ctx, key1.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) + data, err = f.txn.Datastore().Get(f.ctx, key2.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} + +func TestNonUniqueCreate_IfUponIndexingExistingDocsFetcherFails_ReturnError(t *testing.T) { + testError := errors.New("test error") + + cases := []struct { + Name string + PrepareFetcher func() fetcher.Fetcher + }{ + { + Name: "Fails to init", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Unset() + f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testError) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(nil) + return f + }, + }, + { + Name: "Fails to start", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().Start(mock.Anything, mock.Anything, mock.Anything).Unset() + f.EXPECT().Start(mock.Anything, mock.Anything, mock.Anything).Return(testError) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(nil) + return f + }, + }, + { + Name: "Fails to fetch next decoded", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().FetchNextDecoded(mock.Anything).Unset() + f.EXPECT().FetchNextDecoded(mock.Anything).Return(nil, testError) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(nil) + return f + }, + }, + { + Name: "Fails to close", + PrepareFetcher: func() fetcher.Fetcher { + f := fetcherMocks.NewStubbedFetcher(t) + f.EXPECT().FetchNextDecoded(mock.Anything).Unset() + f.EXPECT().FetchNextDecoded(mock.Anything).Return(nil, nil) + f.EXPECT().Close().Unset() + f.EXPECT().Close().Return(testError) + return f + }, + }, + } + + for _, tc := range cases { + f := newIndexTestFixture(t) + + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + + f.users.fetcherFactory = tc.PrepareFetcher + key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + + _, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) + require.ErrorIs(t, err, testError, tc.Name) + + _, err = f.txn.Datastore().Get(f.ctx, key.ToDS()) + require.Error(t, err, tc.Name) + } +} + +func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + + testErr := errors.New("test error") + + f.users.fetcherFactory = func() fetcher.Fetcher { + return fetcherMocks.NewStubbedFetcher(t) + } + + mockedTxn := f.mockTxn() + mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) + mockedTxn.MockDatastore.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Unset() + mockedTxn.MockDatastore.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything). + Return(testErr) + mockedTxn.EXPECT().Datastore().Unset() + mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) + + _, err := f.users.WithTxn(mockedTxn).CreateIndex(f.ctx, getUsersIndexDescOnName()) + require.ErrorIs(f.t, err, testErr) +} + func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f := newIndexTestFixtureBare(t) users := f.createCollection(getUsersCollectionDesc()) From b96c1dff26c104ca71339b5db6e43e3cbdc7e1e8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 5 Jun 2023 12:34:39 +0200 Subject: [PATCH 082/120] Small refactor --- db/collection_index.go | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index e21b9752db..a9caf5ee24 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -228,24 +228,14 @@ func (c *collection) newFetcher() fetcher.Fetcher { return new(fetcher.DocumentFetcher) } } -func (c *collection) indexExistingDocs( + +func (c *collection) iterateAllDocs( ctx context.Context, txn datastore.Txn, - index CollectionIndex, + fields []*client.FieldDescription, + exec func(doc *client.Document) error, ) error { df := c.newFetcher() - - fields := make([]*client.FieldDescription, 0, 1) - for _, field := range index.Description().Fields { - for i := range c.desc.Schema.Fields { - colField := &c.desc.Schema.Fields[i] - if field.Name == colField.Name { - fields = append(fields, colField) - break - } - } - } - err := df.Init(&c.desc, fields, false, false) if err != nil { _ = df.Close() @@ -270,7 +260,7 @@ func (c *collection) indexExistingDocs( if doc == nil { break } - err = index.Save(ctx, txn, doc) + err = exec(doc) if err != nil { return err } @@ -279,6 +269,27 @@ func (c *collection) indexExistingDocs( return df.Close() } +func (c *collection) indexExistingDocs( + ctx context.Context, + txn datastore.Txn, + index CollectionIndex, +) error { + fields := make([]*client.FieldDescription, 0, 1) + for _, field := range index.Description().Fields { + for i := range c.desc.Schema.Fields { + colField := &c.desc.Schema.Fields[i] + if field.Name == colField.Name { + fields = append(fields, colField) + break + } + } + } + + return c.iterateAllDocs(ctx, txn, fields, func(doc *client.Document) error { + return index.Save(ctx, txn, doc) + }) +} + func (c *collection) DropIndex(ctx context.Context, indexName string) error { key := core.NewCollectionIndexKey(c.Name(), indexName) From 042356c0155098c1bffd0e83e4cf9f5192fcab13 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 5 Jun 2023 13:02:21 +0200 Subject: [PATCH 083/120] Add test for updating indexed nil values --- db/indexed_docs_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 45276f456c..0fa1803492 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -981,3 +981,35 @@ func TestNonUniqueUpdate_IfDatastoreFails_ReturnError(t *testing.T) { require.ErrorIs(t, err, testErr) } } + +func TestNonUpdate_IfIndexedFieldWasNil_ShouldDeleteIt(t *testing.T) { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() + + docJSON, err := json.Marshal(struct { + Age int `json:"age"` + }{Age: 44}) + require.NoError(f.t, err) + + doc, err := client.NewDocFromJSON(docJSON) + require.NoError(f.t, err) + + f.saveDocToCollection(doc, f.users) + + oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). + Values(testNilValue).Build() + + err = doc.Set(usersNameFieldName, "John") + require.NoError(f.t, err) + + err = f.users.Update(f.ctx, doc) + require.NoError(f.t, err) + f.commitTxn() + + newKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + + _, err = f.txn.Datastore().Get(f.ctx, newKey.ToDS()) + require.NoError(t, err) + _, err = f.txn.Datastore().Get(f.ctx, oldKey.ToDS()) + require.Error(t, err) +} From 98e3ce7bc029b56dee91d58052cf5affaf33b4ff Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 5 Jun 2023 15:34:33 +0200 Subject: [PATCH 084/120] Add documentation --- client/index.go | 4 ++++ core/key.go | 21 +++++++++++---------- core/key_test.go | 8 ++++---- db/collection_index.go | 2 +- db/index.go | 9 +++++++++ 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/client/index.go b/client/index.go index 3fa829b284..9941f8877f 100644 --- a/client/index.go +++ b/client/index.go @@ -14,13 +14,17 @@ package client type IndexDirection string const ( + // Ascending is the value to use for an ascending fields Ascending IndexDirection = "ASC" + // Descending is the value to use for an descending fields Descending IndexDirection = "DESC" ) // IndexFieldDescription describes how a field is being indexed. type IndexedFieldDescription struct { + // Name contains the name of the field. Name string + // Direction contains the direction of the index. Direction IndexDirection } diff --git a/core/key.go b/core/key.go index 136941b086..fe35d554d5 100644 --- a/core/key.go +++ b/core/key.go @@ -68,7 +68,7 @@ type DataStoreKey struct { var _ Key = (*DataStoreKey)(nil) -// IndexDataStoreKey is a type that represents a key of an indexed document in the database. +// IndexDataStoreKey is key of an indexed document in the database. type IndexDataStoreKey struct { CollectionID string IndexID string @@ -116,9 +116,10 @@ type CollectionSchemaVersionKey struct { var _ Key = (*CollectionSchemaVersionKey)(nil) +// CollectionIndexKey is key for storing an index description type CollectionIndexKey struct { - CollectionName string - IndexName string + CollectionID string + IndexID string } var _ Key = (*CollectionIndexKey)(nil) @@ -228,7 +229,7 @@ func NewCollectionSchemaVersionKey(schemaVersionId string) CollectionSchemaVersi } func NewCollectionIndexKey(colID, name string) CollectionIndexKey { - return CollectionIndexKey{CollectionName: colID, IndexName: name} + return CollectionIndexKey{CollectionID: colID, IndexID: name} } func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { @@ -236,9 +237,9 @@ func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { if len(keyArr) < 4 || len(keyArr) > 5 || keyArr[1] != "collection" || keyArr[2] != "index" { return CollectionIndexKey{}, ErrInvalidKey } - result := CollectionIndexKey{CollectionName: keyArr[3]} + result := CollectionIndexKey{CollectionID: keyArr[3]} if len(keyArr) == 5 { - result.IndexName = keyArr[4] + result.IndexID = keyArr[4] } return result, nil } @@ -532,10 +533,10 @@ func (k CollectionSchemaVersionKey) ToDS() ds.Key { func (k CollectionIndexKey) ToString() string { result := COLLECTION_INDEX - if k.CollectionName != "" { - result = result + "/" + k.CollectionName - if k.IndexName != "" { - result = result + "/" + k.IndexName + if k.CollectionID != "" { + result = result + "/" + k.CollectionID + if k.IndexID != "" { + result = result + "/" + k.IndexID } } diff --git a/core/key_test.go b/core/key_test.go index d0b0fb104f..98a9a04fff 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -146,15 +146,15 @@ func TestNewIndexKeyFromString_IfInvalidString_ReturnError(t *testing.T) { func TestNewIndexKeyFromString_IfOnlyCollectionName_ReturnKey(t *testing.T) { key, err := NewCollectionIndexKeyFromString("/collection/index/col") assert.NoError(t, err) - assert.Equal(t, key.CollectionName, "col") - assert.Equal(t, key.IndexName, "") + assert.Equal(t, key.CollectionID, "col") + assert.Equal(t, key.IndexID, "") } func TestNewIndexKeyFromString_IfFullKeyString_ReturnKey(t *testing.T) { key, err := NewCollectionIndexKeyFromString("/collection/index/col/idx") assert.NoError(t, err) - assert.Equal(t, key.CollectionName, "col") - assert.Equal(t, key.IndexName, "idx") + assert.Equal(t, key.CollectionID, "col") + assert.Equal(t, key.IndexID, "idx") } func TestIndexDatastoreKey_ToString(t *testing.T) { diff --git a/db/collection_index.go b/db/collection_index.go index a9caf5ee24..61032d8a42 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -99,7 +99,7 @@ func (db *db) getAllCollectionIndexes( return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) } indexes = append(indexes, collectionIndexDescription{ - CollectionName: indexKey.CollectionName, + CollectionName: indexKey.CollectionID, Index: colDesk, }) } diff --git a/db/index.go b/db/index.go index f14f9a4ea2..76a713a129 100644 --- a/db/index.go +++ b/db/index.go @@ -30,11 +30,19 @@ const ( indexFieldNilValue = "n" ) +// CollectionIndex is an interface for collection indexes +// It abstracts away common index functionality to be implemented +// by different index types: non-unique, unique, and composite type CollectionIndex interface { + // Save indexes a document by storing it Save(context.Context, datastore.Txn, *client.Document) error + // Update updates an existing document in the index Update(context.Context, datastore.Txn, *client.Document, *client.Document) error + // RemoveAll removes all documents from the index RemoveAll(context.Context, datastore.Txn) error + // Name returns the name of the index Name() string + // Description returns the description of the index Description() client.IndexDescription } @@ -86,6 +94,7 @@ func getFieldValConverter(kind client.FieldKind) func(any) ([]byte, error) { } } +// NewCollectionIndex creates a new collection index func NewCollectionIndex( collection client.Collection, desc client.IndexDescription, From f8cc5e520a831ec1479ef90f3cf9da5944ed236b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 6 Jun 2023 11:46:03 +0200 Subject: [PATCH 085/120] Add documentation --- core/key.go | 114 +++++++++++++++++++++++++---------------- core/key_test.go | 8 +-- db/collection_index.go | 2 +- 3 files changed, 76 insertions(+), 48 deletions(-) diff --git a/core/key.go b/core/key.go index fe35d554d5..65bbc709d8 100644 --- a/core/key.go +++ b/core/key.go @@ -70,9 +70,12 @@ var _ Key = (*DataStoreKey)(nil) // IndexDataStoreKey is key of an indexed document in the database. type IndexDataStoreKey struct { + // CollectionID is the id (unique number) of the collection CollectionID string - IndexID string - FieldValues []string + // IndexID is the id (unique number) of the index + IndexID string + // FieldValues is the values of the fields in the index + FieldValues []string } var _ Key = (*IndexDataStoreKey)(nil) @@ -116,10 +119,12 @@ type CollectionSchemaVersionKey struct { var _ Key = (*CollectionSchemaVersionKey)(nil) -// CollectionIndexKey is key for storing an index description +// CollectionIndexKey to a stored description of an index type CollectionIndexKey struct { - CollectionID string - IndexID string + // CollectionName is the name of the collection that the index is on + CollectionName string + // IndexName is the name of the index + IndexName string } var _ Key = (*CollectionIndexKey)(nil) @@ -228,22 +233,55 @@ func NewCollectionSchemaVersionKey(schemaVersionId string) CollectionSchemaVersi return CollectionSchemaVersionKey{SchemaVersionId: schemaVersionId} } -func NewCollectionIndexKey(colID, name string) CollectionIndexKey { - return CollectionIndexKey{CollectionID: colID, IndexID: name} +func NewCollectionIndexKey(colID, indexName string) CollectionIndexKey { + return CollectionIndexKey{CollectionName: colID, IndexName: indexName} } +// NewCollectionIndexKeyFromString creates a new CollectionIndexKey from a string. +// It expects the input string is in the following format: +// +// /collection/index/[CollectionName]/[IndexName] +// +// Where [IndexName] might be omitted. Anything else will return an error. func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { keyArr := strings.Split(key, "/") if len(keyArr) < 4 || len(keyArr) > 5 || keyArr[1] != "collection" || keyArr[2] != "index" { return CollectionIndexKey{}, ErrInvalidKey } - result := CollectionIndexKey{CollectionID: keyArr[3]} + result := CollectionIndexKey{CollectionName: keyArr[3]} if len(keyArr) == 5 { - result.IndexID = keyArr[4] + result.IndexName = keyArr[4] } return result, nil } +// ToString returns the string representation of the key +// It is in the following format: +// /collection/index/[CollectionName]/[IndexName] +// if [CollectionName] is empty, the rest is ignored +func (k CollectionIndexKey) ToString() string { + result := COLLECTION_INDEX + + if k.CollectionName != "" { + result = result + "/" + k.CollectionName + if k.IndexName != "" { + result = result + "/" + k.IndexName + } + } + + return result +} + +// Bytes returns the byte representation of the key +func (k CollectionIndexKey) Bytes() []byte { + return []byte(k.ToString()) +} + +// ToDS returns the datastore key +func (k CollectionIndexKey) ToDS() ds.Key { + return ds.NewKey(k.ToString()) +} + func NewSequenceKey(name string) SequenceKey { return SequenceKey{SequenceName: name} } @@ -352,35 +390,34 @@ func (k DataStoreKey) ToPrimaryDataStoreKey() PrimaryDataStoreKey { } } -// NewIndexDataStoreKey creates a new IndexDataStoreKey from a string as best as it can, -// splitting the input using '/' as a field deliminator. It assumes -// that the input string is in the following format: +// NewIndexDataStoreKey creates a new IndexDataStoreKey from a string. +// It expects the input string is in the following format: // -// /[CollectionID]/[IndexID]/[FieldID](/[FieldID]...) +// /[CollectionID]/[IndexID]/[FieldValue](/[FieldValue]...) // -// Any properties before the above (assuming a '/' deliminator) are ignored +// Where [CollectionID] and [IndexID] are integers func NewIndexDataStoreKey(key string) (IndexDataStoreKey, error) { - indexKey := IndexDataStoreKey{} if key == "" { - return indexKey, ErrEmptyKey + return IndexDataStoreKey{}, ErrEmptyKey } if !strings.HasPrefix(key, "/") { - return indexKey, ErrInvalidKey + return IndexDataStoreKey{}, ErrInvalidKey } elements := strings.Split(key[1:], "/") // With less than 3 elements, we know it's an invalid key if len(elements) < 3 { - return indexKey, ErrInvalidKey + return IndexDataStoreKey{}, ErrInvalidKey } _, err := strconv.Atoi(elements[0]) if err != nil { return IndexDataStoreKey{}, ErrInvalidKey } - indexKey.CollectionID = elements[0] + + indexKey := IndexDataStoreKey{CollectionID: elements[0]} _, err = strconv.Atoi(elements[1]) if err != nil { @@ -399,14 +436,21 @@ func NewIndexDataStoreKey(key string) (IndexDataStoreKey, error) { return indexKey, nil } +// Bytes returns the byte representation of the key func (k *IndexDataStoreKey) Bytes() []byte { return []byte(k.ToString()) } +// ToDS returns the datastore key func (k *IndexDataStoreKey) ToDS() ds.Key { return ds.NewKey(k.ToString()) } +// ToString returns the string representation of the key +// It is in the following format: +// /[CollectionID]/[IndexID]/[FieldValue](/[FieldValue]...) +// If while composing the string from left to right, a component +// is empty, the string is returned up to that point func (k *IndexDataStoreKey) ToString() string { sb := strings.Builder{} @@ -433,10 +477,15 @@ func (k *IndexDataStoreKey) ToString() string { return sb.String() } +// Equal returns true if the two keys are equal func (k IndexDataStoreKey) Equal(other IndexDataStoreKey) bool { - if k.CollectionID != other.CollectionID || - k.IndexID != other.IndexID || - len(k.FieldValues) != len(other.FieldValues) { + if k.CollectionID != other.CollectionID { + return false + } + if k.IndexID != other.IndexID { + return false + } + if len(k.FieldValues) != len(other.FieldValues) { return false } for i := range k.FieldValues { @@ -530,27 +579,6 @@ func (k CollectionSchemaVersionKey) ToDS() ds.Key { return ds.NewKey(k.ToString()) } -func (k CollectionIndexKey) ToString() string { - result := COLLECTION_INDEX - - if k.CollectionID != "" { - result = result + "/" + k.CollectionID - if k.IndexID != "" { - result = result + "/" + k.IndexID - } - } - - return result -} - -func (k CollectionIndexKey) Bytes() []byte { - return []byte(k.ToString()) -} - -func (k CollectionIndexKey) ToDS() ds.Key { - return ds.NewKey(k.ToString()) -} - func (k SequenceKey) ToString() string { result := SEQ diff --git a/core/key_test.go b/core/key_test.go index 98a9a04fff..d0b0fb104f 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -146,15 +146,15 @@ func TestNewIndexKeyFromString_IfInvalidString_ReturnError(t *testing.T) { func TestNewIndexKeyFromString_IfOnlyCollectionName_ReturnKey(t *testing.T) { key, err := NewCollectionIndexKeyFromString("/collection/index/col") assert.NoError(t, err) - assert.Equal(t, key.CollectionID, "col") - assert.Equal(t, key.IndexID, "") + assert.Equal(t, key.CollectionName, "col") + assert.Equal(t, key.IndexName, "") } func TestNewIndexKeyFromString_IfFullKeyString_ReturnKey(t *testing.T) { key, err := NewCollectionIndexKeyFromString("/collection/index/col/idx") assert.NoError(t, err) - assert.Equal(t, key.CollectionID, "col") - assert.Equal(t, key.IndexID, "idx") + assert.Equal(t, key.CollectionName, "col") + assert.Equal(t, key.IndexName, "idx") } func TestIndexDatastoreKey_ToString(t *testing.T) { diff --git a/db/collection_index.go b/db/collection_index.go index 61032d8a42..a9caf5ee24 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -99,7 +99,7 @@ func (db *db) getAllCollectionIndexes( return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) } indexes = append(indexes, collectionIndexDescription{ - CollectionName: indexKey.CollectionID, + CollectionName: indexKey.CollectionName, Index: colDesk, }) } From ac5328dff310efdc70bc21a93d42259f8652ee17 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 6 Jun 2023 16:37:40 +0200 Subject: [PATCH 086/120] Add integration test --- tests/integration/index/simple_test.go | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/integration/index/simple_test.go diff --git a/tests/integration/index/simple_test.go b/tests/integration/index/simple_test.go new file mode 100644 index 0000000000..bab0772fad --- /dev/null +++ b/tests/integration/index/simple_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestIndexCreate_ShouldNotHinderQuerying(t *testing.T) { + test := testUtils.TestCase{ + Description: "Creation of index should not hinder querying", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + Name: String @index + Age: Int + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + // bae-52b9170d-b77a-5887-b877-cbdbb99b009f + Doc: ` + { + "Name": "John", + "Age": 21 + }`, + }, + testUtils.Request{ + Request: ` + query { + Users { + _key + Name + Age + } + }`, + Results: []map[string]any{ + { + "_key": "bae-52b9170d-b77a-5887-b877-cbdbb99b009f", + "Name": "John", + "Age": uint64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} From 8cf92b6ea31e2f0c04f4fd24156ee99b539ecd5b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 9 Jun 2023 12:07:44 +0200 Subject: [PATCH 087/120] Add integration tests for creating and dropping index --- client/collection.go | 6 +- db/collection.go | 6 + db/collection_index.go | 7 +- db/index_test.go | 12 +- request/graphql/schema/collection.go | 11 +- request/graphql/schema/errors.go | 5 + request/graphql/schema/index_test.go | 10 +- tests/integration/index/create_test.go | 135 ++++++++++++++++++ .../index/{simple_test.go => drop_test.go} | 8 +- tests/integration/test_case.go | 53 +++++++ tests/integration/utils2.go | 125 ++++++++++++++++ 11 files changed, 362 insertions(+), 16 deletions(-) create mode 100644 tests/integration/index/create_test.go rename tests/integration/index/{simple_test.go => drop_test.go} (84%) diff --git a/client/collection.go b/client/collection.go index c1f5e821cf..75d737d2cb 100644 --- a/client/collection.go +++ b/client/collection.go @@ -138,7 +138,11 @@ type Collection interface { GetAllDocKeys(ctx context.Context) (<-chan DocKeysResult, error) // CreateIndex creates a new index on the collection. - CreateIndex(ctx context.Context, desc IndexDescription) (IndexDescription, error) + // `IndexDescription` contains the description of the index to be created. + // `IndexDescription.Name` must start with a letter or an underscore and can + // only contain letters, numbers, and underscores. + // If the name of the index is not provided, it will be generated. + CreateIndex(context.Context, IndexDescription) (IndexDescription, error) // DropIndex drops an index from the collection. DropIndex(ctx context.Context, indexName string) error diff --git a/db/collection.go b/db/collection.go index d66cb3c7b5..0bc409f58d 100644 --- a/db/collection.go +++ b/db/collection.go @@ -191,6 +191,12 @@ func (db *db) createCollection( logging.NewKV("Name", col.Name()), logging.NewKV("SchemaID", col.SchemaID()), ) + + for _, index := range col.desc.Indexes { + if _, err := col.createIndex(ctx, txn, index); err != nil { + return nil, err + } + } return col, nil } diff --git a/db/collection_index.go b/db/collection_index.go index a9caf5ee24..5e841254cb 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -25,6 +25,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/db/base" "github.com/sourcenetwork/defradb/db/fetcher" + "github.com/sourcenetwork/defradb/request/graphql/schema" ) // collectionIndexDescription describes an index on a collection. @@ -372,6 +373,9 @@ func (c *collection) createIndex( txn datastore.Txn, desc client.IndexDescription, ) (CollectionIndex, error) { + if desc.Name != "" && !schema.IsValidIndexName(desc.Name) { + return nil, schema.NewErrIndexWithInvalidName("!") + } err := validateIndexDescription(desc) if err != nil { return nil, err @@ -488,9 +492,6 @@ func validateIndexDescription(desc client.IndexDescription) error { func generateIndexName(col client.Collection, fields []client.IndexedFieldDescription, inc int) string { sb := strings.Builder{} direction := "ASC" - //if fields[0].Direction == client.Descending { - //direction = "DESC" - //} sb.WriteString(col.Name()) sb.WriteByte('_') sb.WriteString(fields[0].Name) diff --git a/db/index_test.go b/db/index_test.go index d9004a5d13..6ab1797571 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -28,6 +28,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/datastore/mocks" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/request/graphql/schema" ) const ( @@ -323,7 +324,7 @@ func TestCreateIndex_IfFieldHasNoDirection_DefaultToAsc(t *testing.T) { assert.Equal(t, client.Ascending, newDesc.Fields[0].Direction) } -func TestCreateIndex_IfNameIsNotSpecified_GenerateWithLowerCase(t *testing.T) { +func TestCreateIndex_IfNameIsNotSpecified_Generate(t *testing.T) { f := newIndexTestFixtureBare(t) colDesc := getUsersCollectionDesc() const colName = "UsErS" @@ -555,6 +556,15 @@ func TestCreateIndex_IfFailsToCreateTxn_ReturnError(t *testing.T) { require.ErrorIs(t, err, testErr) } +func TestCreateIndex_IfProvideInvalidIndexName_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + indexDesc := getUsersIndexDescOnName() + indexDesc.Name = "!" + _, err := f.users.CreateIndex(f.ctx, indexDesc) + require.ErrorIs(t, err, schema.NewErrIndexWithInvalidName(indexDesc.Name)) +} + func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { f := newIndexTestFixture(t) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 51dee05f85..fe84abc6b7 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -151,7 +151,10 @@ func fromAstDefinition( }, nil } -func isValidIndexName(name string) bool { +// IsValidIndexName returns true if the name is a valid index name. +// Valid index names must start with a letter or underscore, and can +// contain letters, numbers, and underscores. +func IsValidIndexName(name string) bool { if len(name) == 0 { return false } @@ -184,8 +187,8 @@ func fieldIndexFromAST(field *ast.FieldDefinition, directive *ast.Directive) (cl return client.IndexDescription{}, ErrIndexWithInvalidArg } desc.Name = nameVal.Value - if !isValidIndexName(desc.Name) { - return client.IndexDescription{}, ErrIndexWithInvalidArg + if !IsValidIndexName(desc.Name) { + return client.IndexDescription{}, NewErrIndexWithInvalidName(desc.Name) } default: return client.IndexDescription{}, ErrIndexWithUnknownArg @@ -205,7 +208,7 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { return client.IndexDescription{}, ErrIndexWithInvalidArg } desc.Name = nameVal.Value - if !isValidIndexName(desc.Name) { + if !IsValidIndexName(desc.Name) { return client.IndexDescription{}, ErrIndexWithInvalidArg } case indexDirectivePropFields: diff --git a/request/graphql/schema/errors.go b/request/graphql/schema/errors.go index 9290436ee5..cf28c7d710 100644 --- a/request/graphql/schema/errors.go +++ b/request/graphql/schema/errors.go @@ -25,6 +25,7 @@ const ( errIndexMissingFields string = "index missing fields" errIndexUnknownArgument string = "index with unknown argument" errIndexInvalidArgument string = "index with invalid argument" + errIndexInvalidName string = "index with invalid name" ) var ( @@ -57,6 +58,10 @@ func NewErrDuplicateField(objectName, fieldName string) error { ) } +func NewErrIndexWithInvalidName(name string) error { + return errors.New(errIndexInvalidName, errors.NewKV("Name", name)) +} + func NewErrFieldMissingRelation(objectName, fieldName string, objectType string) error { return errors.New( errFieldMissingRelation, diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index bd5d3b090d..c37f62e8e6 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -240,28 +240,28 @@ func TestInvalidFieldIndex(t *testing.T) { sdl: `type user { name: String @index(name: "1_user_name") }`, - expectedErr: errIndexInvalidArgument, + expectedErr: errIndexInvalidName, }, { description: "field index with empty name", sdl: `type user { name: String @index(name: "") }`, - expectedErr: errIndexInvalidArgument, + expectedErr: errIndexInvalidName, }, { description: "field index name with spaces", sdl: `type user { name: String @index(name: "user name") }`, - expectedErr: errIndexInvalidArgument, + expectedErr: errIndexInvalidName, }, { description: "field index name with special symbols", sdl: `type user { name: String @index(name: "user!name") }`, - expectedErr: errIndexInvalidArgument, + expectedErr: errIndexInvalidName, }, } @@ -287,7 +287,7 @@ func parseInvalidIndexAndTest(t *testing.T, testCase invalidIndexTestCase) { ctx := context.Background() _, err := FromString(ctx, testCase.sdl) - assert.EqualError(t, err, testCase.expectedErr, testCase.description) + assert.ErrorContains(t, err, testCase.expectedErr, testCase.description) } type indexTestCase struct { diff --git a/tests/integration/index/create_test.go b/tests/integration/index/create_test.go new file mode 100644 index 0000000000..5c58c16bef --- /dev/null +++ b/tests/integration/index/create_test.go @@ -0,0 +1,135 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + "github.com/sourcenetwork/defradb/request/graphql/schema" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestIndexCreateWithCollection_ShouldNotHinderQuerying(t *testing.T) { + test := testUtils.TestCase{ + Description: "Creation of index with collection should not hinder querying", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + Name: String @index + Age: Int + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + // bae-52b9170d-b77a-5887-b877-cbdbb99b009f + Doc: ` + { + "Name": "John", + "Age": 21 + }`, + }, + testUtils.Request{ + Request: ` + query { + Users { + _key + Name + Age + } + }`, + Results: []map[string]any{ + { + "_key": "bae-52b9170d-b77a-5887-b877-cbdbb99b009f", + "Name": "John", + "Age": uint64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} + +func TestIndexCreate_ShouldNotHinderQuerying(t *testing.T) { + test := testUtils.TestCase{ + Description: "Creation of index separately from a collection should not hinder querying", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + Name: String + Age: Int + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + // bae-52b9170d-b77a-5887-b877-cbdbb99b009f + Doc: ` + { + "Name": "John", + "Age": 21 + }`, + }, + testUtils.CreateIndex{ + CollectionID: 0, + IndexName: "some_index", + FieldName: "Name", + }, + testUtils.Request{ + Request: ` + query { + Users { + _key + Name + Age + } + }`, + Results: []map[string]any{ + { + "_key": "bae-52b9170d-b77a-5887-b877-cbdbb99b009f", + "Name": "John", + "Age": uint64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} + +func TestIndexCreate_IfInvalidIndexName_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + Description: "If invalid index name is provided, return error", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + Name: String + Age: Int + } + `, + }, + testUtils.CreateIndex{ + CollectionID: 0, + IndexName: "!", + FieldName: "Name", + ExpectedError: schema.NewErrIndexWithInvalidName("!").Error(), + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} diff --git a/tests/integration/index/simple_test.go b/tests/integration/index/drop_test.go similarity index 84% rename from tests/integration/index/simple_test.go rename to tests/integration/index/drop_test.go index bab0772fad..31e2e90b56 100644 --- a/tests/integration/index/simple_test.go +++ b/tests/integration/index/drop_test.go @@ -16,9 +16,9 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -func TestIndexCreate_ShouldNotHinderQuerying(t *testing.T) { +func TestIndexDrop_ShouldNotHinderQuerying(t *testing.T) { test := testUtils.TestCase{ - Description: "Creation of index should not hinder querying", + Description: "Creation of index with collection should not hinder querying", Actions: []any{ testUtils.SchemaUpdate{ Schema: ` @@ -37,6 +37,10 @@ func TestIndexCreate_ShouldNotHinderQuerying(t *testing.T) { "Age": 21 }`, }, + testUtils.DropIndex{ + CollectionID: 0, + IndexID: 0, + }, testUtils.Request{ Request: ` query { diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 9a8f2300df..a40ba591fc 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -13,6 +13,7 @@ package tests import ( "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/config" ) @@ -151,6 +152,58 @@ type UpdateDoc struct { DontSync bool } +// CreateIndex will attempt to create the given secondary index for the given collection +// using the collection api. +type CreateIndex struct { + // NodeID may hold the ID (index) of a node to create the secondary index on. + // + // If a value is not provided the index will be created in all nodes. + NodeID immutable.Option[int] + + // The collection for which this index should be created. + CollectionID int + + // The name of the index to create. If not provided, one will be generated. + IndexName string + + // The name of the field to index. Used only for single field indexes. + FieldName string + + // The names of the fields to index. Used only for composite indexes. + FieldsNames []string + // The directions of the 'FieldsNames' to index. Used only for composite indexes. + Directions []client.IndexDirection + + // Any error expected from the action. Optional. + // + // String can be a partial, and the test will pass if an error is returned that + // contains this string. + ExpectedError string +} + +// DropIndex will attempt to drop the given secondary index from the given collection +// using the collection api. +type DropIndex struct { + // NodeID may hold the ID (index) of a node to delete the secondary index from. + // + // If a value is not provided the index will be deleted from all nodes. + NodeID immutable.Option[int] + + // The collection from which the index should be deleted. + CollectionID int + + // The index-identifier of the secondary index within the collection. + // This is based on the order in which it was created, not the ordering of + // the indexes within the database. + IndexID int + + // Any error expected from the action. Optional. + // + // String can be a partial, and the test will pass if an error is returned that + // contains this string. + ExpectedError string +} + // Request represents a standard Defra (GQL) request. type Request struct { // NodeID may hold the ID (index) of a node to execute this request on. diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 1f55bb439d..4621da9aa2 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -314,6 +314,8 @@ func executeTestCase( collections := getCollections(ctx, t, nodes, collectionNames) // documents are by collection (index), these are not node specific. documents := getDocuments(ctx, t, testCase, collections, startActionIndex) + // indexes are by collection (index) + indexes := getIndexes(ctx, collections) for i := startActionIndex; i <= endActionIndex; i++ { // declare default database for ease of use @@ -347,6 +349,7 @@ func executeTestCase( // If the db was restarted we need to refresh the collection definitions as the old instances // will reference the old (closed) database instances. collections = getCollections(ctx, t, nodes, collectionNames) + indexes = getIndexes(ctx, collections) case ConnectPeers: syncChans = append(syncChans, connectPeers(ctx, t, testCase, action, nodes, nodeAddresses)) @@ -367,11 +370,13 @@ func executeTestCase( updateSchema(ctx, t, nodes, testCase, action) // If the schema was updated we need to refresh the collection definitions. collections = getCollections(ctx, t, nodes, collectionNames) + indexes = getIndexes(ctx, collections) case SchemaPatch: patchSchema(ctx, t, nodes, testCase, action) // If the schema was updated we need to refresh the collection definitions. collections = getCollections(ctx, t, nodes, collectionNames) + indexes = getIndexes(ctx, collections) case CreateDoc: documents = createDoc(ctx, t, testCase, nodes, collections, documents, action) @@ -382,6 +387,12 @@ func executeTestCase( case UpdateDoc: updateDoc(ctx, t, testCase, nodes, collections, documents, action) + case CreateIndex: + indexes = createIndex(ctx, t, testCase, nodes, collections, indexes, action) + + case DropIndex: + dropIndex(ctx, t, testCase, nodes, collections, indexes, action) + case TransactionRequest2: txns = executeTransactionRequest(ctx, t, db, txns, testCase, action) @@ -819,6 +830,35 @@ func getDocuments( return documentsByCollection } +func getIndexes( + ctx context.Context, + collections [][]client.Collection, +) [][][]client.IndexDescription { + if len(collections) == 0 { + return [][][]client.IndexDescription{} + } + + result := make([][][]client.IndexDescription, len(collections)) + + for i, nodeCols := range collections { + result[i] = make([][]client.IndexDescription, len(nodeCols)) + + for j, col := range nodeCols { + if col == nil { + continue + } + colIndexes, err := col.GetIndexes(ctx) + if err != nil { + continue + } + + result[i][j] = colIndexes + } + } + + return result +} + // updateSchema updates the schema using the given details. func updateSchema( ctx context.Context, @@ -954,6 +994,91 @@ func updateDoc( assertExpectedErrorRaised(t, testCase.Description, action.ExpectedError, expectedErrorRaised) } +// createIndex creates a secondary index using the collection api. +func createIndex( + ctx context.Context, + t *testing.T, + testCase TestCase, + nodes []*node.Node, + nodeCollections [][]client.Collection, + indexes [][][]client.IndexDescription, + action CreateIndex, +) [][][]client.IndexDescription { + if action.CollectionID >= len(indexes) { + // Expand the slice if required, so that the index can be accessed by collection index + indexes = append(indexes, + make([][][]client.IndexDescription, action.CollectionID-len(indexes)+1)...) + } + actionNodes := getNodes(action.NodeID, nodes) + for nodeID, collections := range getNodeCollections(action.NodeID, nodeCollections) { + indexDesc := client.IndexDescription{ + Name: action.IndexName, + } + if action.FieldName != "" { + indexDesc.Fields = []client.IndexedFieldDescription{ + { + Name: action.FieldName, + }, + } + } else if len(action.FieldsNames) > 0 { + for i := range action.FieldsNames { + indexDesc.Fields = append(indexDesc.Fields, client.IndexedFieldDescription{ + Name: action.FieldsNames[i], + Direction: action.Directions[i], + }) + } + } + err := withRetry( + actionNodes, + nodeID, + func() error { + desc, err := collections[action.CollectionID].CreateIndex(ctx, indexDesc) + if err != nil { + return err + } + indexes[nodeID][action.CollectionID] = + append(indexes[nodeID][action.CollectionID], desc) + return nil + }, + ) + if AssertError(t, testCase.Description, err, action.ExpectedError) { + return nil + } + } + + assertExpectedErrorRaised(t, testCase.Description, action.ExpectedError, false) + + return indexes +} + +// dropIndex drops the secondary index using the collection api. +func dropIndex( + ctx context.Context, + t *testing.T, + testCase TestCase, + nodes []*node.Node, + nodeCollections [][]client.Collection, + indexes [][][]client.IndexDescription, + action DropIndex, +) { + var expectedErrorRaised bool + actionNodes := getNodes(action.NodeID, nodes) + for nodeID, collections := range getNodeCollections(action.NodeID, nodeCollections) { + indexDesc := indexes[nodeID][action.CollectionID][action.IndexID] + + err := withRetry( + actionNodes, + nodeID, + func() error { + return collections[action.CollectionID].DropIndex(ctx, indexDesc.Name) + }, + ) + expectedErrorRaised = AssertError(t, testCase.Description, err, action.ExpectedError) + } + + assertExpectedErrorRaised(t, testCase.Description, action.ExpectedError, expectedErrorRaised) +} + // withRetry attempts to perform the given action, retrying up to a DB-defined // maximum attempt count if a transaction conflict error is returned. // From d52f3093c09443e5c19dc6804d2f520801a515b5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 9 Jun 2023 12:43:07 +0200 Subject: [PATCH 088/120] Extract validation of collection fields update --- db/collection.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/db/collection.go b/db/collection.go index 0bc409f58d..893dd3d8fe 100644 --- a/db/collection.go +++ b/db/collection.go @@ -191,7 +191,7 @@ func (db *db) createCollection( logging.NewKV("Name", col.Name()), logging.NewKV("SchemaID", col.SchemaID()), ) - + for _, index := range col.desc.Indexes { if _, err := col.createIndex(ctx, txn, index); err != nil { return nil, err @@ -287,7 +287,6 @@ func (db *db) validateUpdateCollection( txn datastore.Txn, proposedDesc client.CollectionDescription, ) (bool, error) { - var hasChanged bool existingCollection, err := db.getCollectionByName(ctx, txn, proposedDesc.Name) if err != nil { if errors.Is(err, ds.ErrNotFound) { @@ -321,6 +320,20 @@ func (db *db) validateUpdateCollection( return false, ErrCannotSetVersionID } + // If the field is new, then the collection has changed + hasChanged, err := validateUpdateCollectionFields(existingDesc, proposedDesc) + if err != nil { + return hasChanged, err + } + + return hasChanged, nil +} + +func validateUpdateCollectionFields( + existingDesc client.CollectionDescription, + proposedDesc client.CollectionDescription, +) (bool, error) { + hasChanged := false existingFieldsByID := map[client.FieldID]client.FieldDescription{} existingFieldIndexesByName := map[string]int{} for i, field := range existingDesc.Schema.Fields { @@ -342,7 +355,6 @@ func (db *db) validateUpdateCollection( return false, NewErrCannotSetFieldID(proposedField.Name, proposedField.ID) } - // If the field is new, then the collection has changed hasChanged = hasChanged || !fieldAlreadyExists if !fieldAlreadyExists && (proposedField.Kind == client.FieldKind_FOREIGN_OBJECT || @@ -376,7 +388,6 @@ func (db *db) validateUpdateCollection( return false, NewErrCannotDeleteField(field.Name, field.ID) } } - return hasChanged, nil } From 3c0ab14f15eed8ffad9c58b996de10b669865e4f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 12 Jun 2023 15:11:34 +0200 Subject: [PATCH 089/120] Check for index patching Make collection.Description() include indexes' descriptions --- db/collection.go | 48 ++++++- db/collection_index.go | 52 +++---- db/errors.go | 38 +++++- db/index_test.go | 103 +++++++++++++- db/indexed_docs_test.go | 9 +- .../schema/updates/index/simple_test.go | 127 ++++++++++++++++++ 6 files changed, 331 insertions(+), 46 deletions(-) create mode 100644 tests/integration/schema/updates/index/simple_test.go diff --git a/db/collection.go b/db/collection.go index 893dd3d8fe..c39ad0f0c0 100644 --- a/db/collection.go +++ b/db/collection.go @@ -99,8 +99,12 @@ func (db *db) newCollection(desc client.CollectionDescription) (*collection, err } return &collection{ - db: db, - desc: desc, + db: db, + desc: client.CollectionDescription{ + ID: desc.ID, + Name: desc.Name, + Schema: desc.Schema, + }, colID: desc.ID, }, nil } @@ -192,7 +196,7 @@ func (db *db) createCollection( logging.NewKV("SchemaID", col.SchemaID()), ) - for _, index := range col.desc.Indexes { + for _, index := range desc.Indexes { if _, err := col.createIndex(ctx, txn, index); err != nil { return nil, err } @@ -320,13 +324,13 @@ func (db *db) validateUpdateCollection( return false, ErrCannotSetVersionID } - // If the field is new, then the collection has changed - hasChanged, err := validateUpdateCollectionFields(existingDesc, proposedDesc) + hasChangedFields, err := validateUpdateCollectionFields(existingDesc, proposedDesc) if err != nil { - return hasChanged, err + return hasChangedFields, err } - return hasChanged, nil + hasChangedIndexes, err := validateUpdateCollectionIndexes(existingDesc.Indexes, proposedDesc.Indexes) + return hasChangedFields || hasChangedIndexes, err } func validateUpdateCollectionFields( @@ -355,6 +359,7 @@ func validateUpdateCollectionFields( return false, NewErrCannotSetFieldID(proposedField.Name, proposedField.ID) } + // If the field is new, then the collection has changed hasChanged = hasChanged || !fieldAlreadyExists if !fieldAlreadyExists && (proposedField.Kind == client.FieldKind_FOREIGN_OBJECT || @@ -391,6 +396,29 @@ func validateUpdateCollectionFields( return hasChanged, nil } +func validateUpdateCollectionIndexes( + existingIndexes []client.IndexDescription, + proposedIndexes []client.IndexDescription, +) (bool, error) { + existingNameToIndex := map[string]client.IndexDescription{} + for _, index := range existingIndexes { + existingNameToIndex[index.Name] = index + } + for _, proposedIndex := range proposedIndexes { + if _, exists := existingNameToIndex[proposedIndex.Name]; exists { + delete(existingNameToIndex, proposedIndex.Name) + } else { + return false, NewErrCannotAddIndexWithPatch(proposedIndex.Name) + } + } + if len(existingNameToIndex) > 0 { + for _, index := range existingNameToIndex { + return false, NewErrCannotDropIndexWithPatch(index.Name) + } + } + return false, nil +} + // getCollectionByVersionId returns the [*collection] at the given [schemaVersionId] version. // // Will return an error if the given key is empty, or not found. @@ -414,6 +442,12 @@ func (db *db) getCollectionByVersionID( if err != nil { return nil, err } + + indexes, err := db.getCollectionIndexes(ctx, txn, desc.Name) + if err != nil { + return nil, err + } + desc.Indexes = indexes return &collection{ db: db, diff --git a/db/collection_index.go b/db/collection_index.go index 5e841254cb..32c2ff714e 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -47,7 +47,7 @@ func (db *db) createCollectionIndex( ) (client.IndexDescription, error) { col, err := db.getCollectionByName(ctx, txn, collectionName) if err != nil { - return client.IndexDescription{}, NewErrCollectionDoesntExist(collectionName) + return client.IndexDescription{}, NewErrCanNotReadCollection(collectionName, err) } col = col.WithTxn(txn) return col.CreateIndex(ctx, desc) @@ -60,7 +60,7 @@ func (db *db) dropCollectionIndex( ) error { col, err := db.getCollectionByName(ctx, txn, collectionName) if err != nil { - return NewErrCollectionDoesntExist(collectionName) + return NewErrCanNotReadCollection(collectionName, err) } col = col.WithTxn(txn) return col.DropIndex(ctx, indexName) @@ -114,33 +114,7 @@ func (db *db) getCollectionIndexes( colName string, ) ([]client.IndexDescription, error) { prefix := core.NewCollectionIndexKey(colName, "") - q, err := txn.Systemstore().Query(ctx, query.Query{ - Prefix: prefix.ToString(), - }) - if err != nil { - return nil, NewErrFailedToCreateCollectionQuery(err) - } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() - - indexes := make([]client.IndexDescription, 0) - for res := range q.Next() { - if res.Error != nil { - return nil, res.Error - } - - var colDesk client.IndexDescription - err = json.Unmarshal(res.Value, &colDesk) - if err != nil { - return nil, NewErrInvalidStoredIndex(err) - } - indexes = append(indexes, colDesk) - } - - return indexes, nil + return deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) } func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { @@ -302,6 +276,7 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { if err != nil { return err } + var didFind bool for i := range c.indexes { if c.indexes[i].Name() == indexName { err = c.indexes[i].RemoveAll(ctx, txn) @@ -309,6 +284,17 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { return err } c.indexes = append(c.indexes[:i], c.indexes[i+1:]...) + didFind = true + break + } + } + if !didFind { + return NewErrIndexWithNameDoesNotExists(indexName) + } + + for i := range c.desc.Indexes { + if c.desc.Indexes[i].Name == indexName { + c.desc.Indexes = append(c.desc.Indexes[:i], c.desc.Indexes[i+1:]...) break } } @@ -346,6 +332,11 @@ func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]Colle colIndexes = append(colIndexes, NewCollectionIndex(c, index)) } + descriptions := make([]client.IndexDescription, 0, len(colIndexes)) + for _, index := range colIndexes { + descriptions = append(descriptions, index.Description()) + } + c.desc.Indexes = descriptions c.indexes = colIndexes c.isIndexCached = true return colIndexes, nil @@ -411,6 +402,7 @@ func (c *collection) createIndex( return nil, err } colIndex := NewCollectionIndex(c, desc) + c.desc.Indexes = append(c.desc.Indexes, colIndex.Description()) return colIndex, nil } @@ -462,7 +454,7 @@ func (c *collection) processIndexName( return core.CollectionIndexKey{}, err } if exists { - return core.CollectionIndexKey{}, ErrIndexWithNameAlreadyExists + return core.CollectionIndexKey{}, NewErrIndexWithNameAlreadyExists(desc.Name) } } return indexKey, nil diff --git a/db/errors.go b/db/errors.go index 5cc936ead9..30250ca8ee 100644 --- a/db/errors.go +++ b/db/errors.go @@ -51,6 +51,9 @@ const ( errFailedToReadStoredIndexDesc string = "failed to read stored index description" errCanNotIndexInvalidFieldValue string = "can not index invalid field value" errCanNotDeleteIndexedField string = "can not delete indexed field" + errCanNotAddIndexWithPatch string = "adding indexes via patch is not supported" + errCanNotDropIndexWithPatch string = "dropping indexes via patch is not supported" + errIndexWithNameDoesNotExists string = "index with name doesn't exists" ) var ( @@ -102,7 +105,6 @@ var ( ErrIndexFieldMissingName = errors.New(errIndexFieldMissingName) ErrIndexFieldMissingDirection = errors.New(errIndexFieldMissingDirection) ErrIndexSingleFieldWrongDirection = errors.New(errIndexSingleFieldWrongDirection) - ErrIndexWithNameAlreadyExists = errors.New(errIndexWithNameAlreadyExists) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document @@ -135,9 +137,9 @@ func NewErrNonExistingFieldForIndex(field string) error { return errors.New(errNonExistingFieldForIndex, errors.NewKV("Field", field)) } -// NewErrCollectionDoesntExist returns a new error indicating the collection doesn't exist. -func NewErrCollectionDoesntExist(colName string) error { - return errors.New(errCollectionDoesntExisting, errors.NewKV("Collection", colName)) +// NewErrCanNotReadCollection returns a new error indicating the collection doesn't exist. +func NewErrCanNotReadCollection(colName string, inner error) error { + return errors.Wrap(errCollectionDoesntExisting, inner, errors.NewKV("Collection", colName)) } // NewErrFailedToStoreIndexedField returns a new error indicating that the indexed field could not be stored. @@ -299,3 +301,31 @@ func NewErrDocumentDeleted(dockey string) error { errors.NewKV("DocKey", dockey), ) } + +func NewErrIndexWithNameAlreadyExists(indexName string) error { + return errors.New( + errIndexWithNameAlreadyExists, + errors.NewKV("Name", indexName), + ) +} + +func NewErrIndexWithNameDoesNotExists(indexName string) error { + return errors.New( + errIndexWithNameDoesNotExists, + errors.NewKV("Name", indexName), + ) +} + +func NewErrCannotAddIndexWithPatch(proposedName string) error { + return errors.New( + errCanNotAddIndexWithPatch, + errors.NewKV("ProposedName", proposedName), + ) +} + +func NewErrCannotDropIndexWithPatch(indexName string) error { + return errors.New( + errCanNotDropIndexWithPatch, + errors.NewKV("Name", indexName), + ) +} diff --git a/db/index_test.go b/db/index_test.go index 6ab1797571..53a6f3f03f 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -373,7 +373,7 @@ func TestCreateIndex_IfIndexWithNameAlreadyExists_ReturnError(t *testing.T) { _, err := f.createCollectionIndex(desc1) assert.NoError(t, err) _, err = f.createCollectionIndex(desc2) - assert.EqualError(t, err, errIndexWithNameAlreadyExists) + assert.ErrorIs(t, err, NewErrIndexWithNameAlreadyExists(name)) } func TestCreateIndex_IfGeneratedNameMatchesExisting_AddIncrement(t *testing.T) { @@ -498,7 +498,7 @@ func TestCreateIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { } _, err := f.createCollectionIndexFor(productsColName, desc) - assert.ErrorIs(t, err, NewErrCollectionDoesntExist(usersColName)) + assert.ErrorIs(t, err, NewErrCanNotReadCollection(usersColName, nil)) } func TestCreateIndex_IfPropertyDoesntExist_ReturnError(t *testing.T) { @@ -565,6 +565,72 @@ func TestCreateIndex_IfProvideInvalidIndexName_ReturnError(t *testing.T) { require.ErrorIs(t, err, schema.NewErrIndexWithInvalidName(indexDesc.Name)) } +func TestCreateIndex_ShouldUpdateCollectionsDescription(t *testing.T) { + f := newIndexTestFixture(t) + + indOnName, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) + require.NoError(t, err) + + assert.ElementsMatch(t, []client.IndexDescription{indOnName}, f.users.Description().Indexes) + + indOnAge, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnAge()) + require.NoError(t, err) + + assert.ElementsMatch(t, []client.IndexDescription{indOnName, indOnAge}, + f.users.Description().Indexes) +} + +func TestCreateIndex_NewCollectionDescription_ShouldIncludeIndexDescription(t *testing.T) { + f := newIndexTestFixture(t) + + _, err := f.createCollectionIndex(getUsersIndexDescOnName()) + require.NoError(t, err) + + desc := getUsersIndexDescOnAge() + desc.Name = "" + _, err = f.createCollectionIndex(desc) + require.NoError(t, err) + + cols, err := f.db.getAllCollections(f.ctx, f.txn) + require.NoError(t, err) + + require.Equal(t, 1, len(cols)) + col := cols[0] + require.Equal(t, 2, len(col.Description().Indexes)) + require.NotEmpty(t, col.Description().Indexes[0].Name) + require.NotEmpty(t, col.Description().Indexes[1].Name) +} + +func TestCreateIndex_IfFailedToReadIndexUponRetrievingCollectionDesc_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + testErr := errors.New("test error") + + mockedTxn := f.mockTxn().ClearSystemStore() + onSystemStore := mockedTxn.MockSystemstore.EXPECT() + + colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, "") + matchPrefixFunc := func(q query.Query) bool { + res := q.Prefix == colIndexKey.ToDS().String() + return res + } + + onSystemStore.Query(mock.Anything, mock.MatchedBy(matchPrefixFunc)).Return(nil, testErr) + + descData, err := json.Marshal(getUsersCollectionDesc()) + require.NoError(t, err) + + onSystemStore.Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithValues(t, []byte("schemaID")), nil) + onSystemStore.Get(mock.Anything, mock.Anything).Unset() + onSystemStore.Get(mock.Anything, mock.Anything).Return(descData, nil) + + f.stubSystemStore(onSystemStore) + + _, err = f.db.getAllCollections(f.ctx, f.txn) + require.ErrorIs(t, err, testErr) +} + func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { f := newIndexTestFixture(t) @@ -1010,7 +1076,7 @@ func TestDropIndex_IfCollectionDoesntExist_ReturnError(t *testing.T) { f := newIndexTestFixture(t) err := f.dropIndex(productsColName, "any_name") - assert.ErrorIs(t, err, NewErrCollectionDoesntExist(usersColName)) + assert.ErrorIs(t, err, NewErrCanNotReadCollection(usersColName, nil)) } func TestDropIndex_IfFailsToQuerySystemStorage_ReturnError(t *testing.T) { @@ -1061,7 +1127,36 @@ func TestDropIndex_IfFailsToDeleteFromStorage_ShouldNotCache(t *testing.T) { require.ErrorIs(t, err, testErr) } -func TestDropAllIndex_ShouldDeleteAllIndexes(t *testing.T) { +func TestDropIndex_ShouldUpdateCollectionsDescription(t *testing.T) { + f := newIndexTestFixture(t) + col := f.users.WithTxn(f.txn) + _, err := col.CreateIndex(f.ctx, getUsersIndexDescOnName()) + require.NoError(t, err) + indOnAge, err := col.CreateIndex(f.ctx, getUsersIndexDescOnAge()) + require.NoError(t, err) + f.commitTxn() + + err = f.users.DropIndex(f.ctx, testUsersColIndexName) + require.NoError(t, err) + + assert.ElementsMatch(t, []client.IndexDescription{indOnAge}, + f.users.Description().Indexes) + + err = f.users.DropIndex(f.ctx, testUsersColIndexAge) + require.NoError(t, err) + + assert.ElementsMatch(t, []client.IndexDescription{}, f.users.Description().Indexes) +} + +func TestDropIndex_IfIndexWithNameDoesNotExist_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + const name = "not_existing_index" + err := f.users.DropIndex(f.ctx, name) + require.ErrorIs(t, err, NewErrIndexWithNameDoesNotExists(name)) +} + +func TestDropAllIndexes_ShouldDeleteAllIndexes(t *testing.T) { f := newIndexTestFixture(t) _, err := f.createCollectionIndexFor(usersColName, client.IndexDescription{ Fields: []client.IndexedFieldDescription{ diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 0fa1803492..e1aaf12a10 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -11,6 +11,7 @@ package db import ( + "context" "encoding/json" "errors" "fmt" @@ -211,8 +212,14 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E require.NoError(f.t, err) colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, "") - matchPrefixFunc := func(q query.Query) bool { return q.Prefix == colIndexKey.ToDS().String() } + matchPrefixFunc := func(q query.Query) bool { + return q.Prefix == colIndexKey.ToDS().String() + } + systemStoreOn.Query(mock.Anything, mock.MatchedBy(matchPrefixFunc)). + RunAndReturn(func(context.Context, query.Query) (query.Results, error) { + return mocks.NewQueryResultsWithValues(f.t, indexOnNameDescData), nil + }).Maybe() systemStoreOn.Query(mock.Anything, mock.MatchedBy(matchPrefixFunc)).Maybe(). Return(mocks.NewQueryResultsWithValues(f.t, indexOnNameDescData), nil) systemStoreOn.Query(mock.Anything, mock.Anything).Maybe(). diff --git a/tests/integration/schema/updates/index/simple_test.go b/tests/integration/schema/updates/index/simple_test.go new file mode 100644 index 0000000000..b54586e0af --- /dev/null +++ b/tests/integration/schema/updates/index/simple_test.go @@ -0,0 +1,127 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestPatching_ForCollectionWithIndex_StillWorks(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, add field", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String @index + age: Int @index + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": 11} } + ] + `, + }, + testUtils.Request{ + Request: `query { + Users { + name + age + email + } + }`, + Results: []map[string]any{}, + }, + }, + } + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} + +func TestPatching_IfAttemptToAddIndex_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test adding index to collection fails", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String @index + age: Int + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Indexes/-", "value": { + "Name": "some_index", + "ID": 0, + "Fields": [ + { + "Name": "age", + "Direction": "ASC" + } + ] + } + } + ] + `, + ExpectedError: "adding indexes via patch is not supported. ProposedName: some_index", + }, + testUtils.Request{ + Request: `query { + Users { + name + } + }`, + Results: []map[string]any{}, + }, + }, + } + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} + +func TestPatching_IfAttemptToDropIndex_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test adding index to collection fails", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String @index + age: Int @index(name: "users_age_index") + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "remove", "path": "/Users/Indexes/1" } + ] + `, + ExpectedError: "dropping indexes via patch is not supported. Name: users_age_index", + }, + testUtils.Request{ + Request: `query { + Users { + name + } + }`, + Results: []map[string]any{}, + }, + }, + } + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} From 30456c19d809ce14d8e7dd5d567406af27a6d223 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 12 Jun 2023 15:58:22 +0200 Subject: [PATCH 090/120] Add check if patches change index fields --- db/collection.go | 12 +- db/errors.go | 2 + .../schema/updates/index/simple_test.go | 112 +++++++++++++++++- 3 files changed, 121 insertions(+), 5 deletions(-) diff --git a/db/collection.go b/db/collection.go index c39ad0f0c0..f424e000da 100644 --- a/db/collection.go +++ b/db/collection.go @@ -405,7 +405,15 @@ func validateUpdateCollectionIndexes( existingNameToIndex[index.Name] = index } for _, proposedIndex := range proposedIndexes { - if _, exists := existingNameToIndex[proposedIndex.Name]; exists { + if existingIndex, exists := existingNameToIndex[proposedIndex.Name]; exists { + if len(existingIndex.Fields) != len(proposedIndex.Fields) { + return false, ErrCanNotChangeIndexWithPatch + } + for i := range existingIndex.Fields { + if existingIndex.Fields[i] != proposedIndex.Fields[i] { + return false, ErrCanNotChangeIndexWithPatch + } + } delete(existingNameToIndex, proposedIndex.Name) } else { return false, NewErrCannotAddIndexWithPatch(proposedIndex.Name) @@ -442,7 +450,7 @@ func (db *db) getCollectionByVersionID( if err != nil { return nil, err } - + indexes, err := db.getCollectionIndexes(ctx, txn, desc.Name) if err != nil { return nil, err diff --git a/db/errors.go b/db/errors.go index 30250ca8ee..6c35f6bc73 100644 --- a/db/errors.go +++ b/db/errors.go @@ -53,6 +53,7 @@ const ( errCanNotDeleteIndexedField string = "can not delete indexed field" errCanNotAddIndexWithPatch string = "adding indexes via patch is not supported" errCanNotDropIndexWithPatch string = "dropping indexes via patch is not supported" + errCanNotChangeIndexWithPatch string = "changing indexes via patch is not supported" errIndexWithNameDoesNotExists string = "index with name doesn't exists" ) @@ -105,6 +106,7 @@ var ( ErrIndexFieldMissingName = errors.New(errIndexFieldMissingName) ErrIndexFieldMissingDirection = errors.New(errIndexFieldMissingDirection) ErrIndexSingleFieldWrongDirection = errors.New(errIndexSingleFieldWrongDirection) + ErrCanNotChangeIndexWithPatch = errors.New(errCanNotChangeIndexWithPatch) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document diff --git a/tests/integration/schema/updates/index/simple_test.go b/tests/integration/schema/updates/index/simple_test.go index b54586e0af..a4c8de6829 100644 --- a/tests/integration/schema/updates/index/simple_test.go +++ b/tests/integration/schema/updates/index/simple_test.go @@ -18,7 +18,7 @@ import ( func TestPatching_ForCollectionWithIndex_StillWorks(t *testing.T) { test := testUtils.TestCase{ - Description: "Test schema update, add field", + Description: "Test patching schema for collection with index still works", Actions: []any{ testUtils.SchemaUpdate{ Schema: ` @@ -52,7 +52,7 @@ func TestPatching_ForCollectionWithIndex_StillWorks(t *testing.T) { func TestPatching_IfAttemptToAddIndex_ReturnError(t *testing.T) { test := testUtils.TestCase{ - Description: "Test adding index to collection fails", + Description: "Test adding index to collection via patch fails", Actions: []any{ testUtils.SchemaUpdate{ Schema: ` @@ -95,7 +95,7 @@ func TestPatching_IfAttemptToAddIndex_ReturnError(t *testing.T) { func TestPatching_IfAttemptToDropIndex_ReturnError(t *testing.T) { test := testUtils.TestCase{ - Description: "Test adding index to collection fails", + Description: "Test dropping index from collection via patch fails", Actions: []any{ testUtils.SchemaUpdate{ Schema: ` @@ -125,3 +125,109 @@ func TestPatching_IfAttemptToDropIndex_ReturnError(t *testing.T) { } testUtils.ExecuteTestCase(t, []string{"Users"}, test) } + +func TestPatching_IfAttemptToChangeIndexName_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test changing index's name via patch fails", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String @index + age: Int + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "replace", "path": "/Users/Indexes/0/Name", "value": "new_index_name" } + ] + `, + ExpectedError: "adding indexes via patch is not supported. ProposedName: new_index_name", + }, + testUtils.Request{ + Request: `query { + Users { + name + } + }`, + Results: []map[string]any{}, + }, + }, + } + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} + +func TestPatching_IfAttemptToChangeIndexField_ReturnError(t *testing.T) { + testCases := []struct { + description string + patch string + }{ + { + description: "Test adding a field to an index via patch fails", + patch: ` + [ + { "op": "add", "path": "/Users/Indexes/0/Fields/-", "value": { + "Name": "age", + "Direction": "ASC" + } + } + ] + `, + }, + { + description: "Test removing a field from an index via patch fails", + patch: ` + [ + { "op": "remove", "path": "/Users/Indexes/0/Fields/0" } + ] + `, + }, + { + description: "Test changing index's field name via patch fails", + patch: ` + [ + { "op": "replace", "path": "/Users/Indexes/0/Fields/0/Name", "value": "new_field_name" } + ] + `, + }, + { + description: "Test changing index's field direction via patch fails", + patch: ` + [ + { "op": "replace", "path": "/Users/Indexes/0/Fields/0/Direction", "value": "DESC" } + ] + `, + }, + } + + for _, testCase := range testCases { + test := testUtils.TestCase{ + Description: testCase.description, + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String @index + age: Int + } + `, + }, + testUtils.SchemaPatch{ + Patch: testCase.patch, + ExpectedError: "changing indexes via patch is not supported", + }, + testUtils.Request{ + Request: `query { + Users { + name + } + }`, + Results: []map[string]any{}, + }, + }, + } + testUtils.ExecuteTestCase(t, []string{"Users"}, test) + } +} From 93458707d3c24fedc646b8d939037fb45bc1f016 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 12 Jun 2023 17:48:34 +0200 Subject: [PATCH 091/120] Switch index values from string to []byte --- core/key.go | 10 ++++---- core/key_test.go | 52 ++++++++++++++++++++++++----------------- db/index.go | 9 +++---- db/indexed_docs_test.go | 21 +++++++++-------- 4 files changed, 51 insertions(+), 41 deletions(-) diff --git a/core/key.go b/core/key.go index 65bbc709d8..ad49fb84a3 100644 --- a/core/key.go +++ b/core/key.go @@ -75,7 +75,7 @@ type IndexDataStoreKey struct { // IndexID is the id (unique number) of the index IndexID string // FieldValues is the values of the fields in the index - FieldValues []string + FieldValues [][]byte } var _ Key = (*IndexDataStoreKey)(nil) @@ -430,7 +430,7 @@ func NewIndexDataStoreKey(key string) (IndexDataStoreKey, error) { if err != nil { return IndexDataStoreKey{}, ErrInvalidKey } - indexKey.FieldValues = append(indexKey.FieldValues, elements[i]) + indexKey.FieldValues = append(indexKey.FieldValues, []byte(elements[i])) } return indexKey, nil @@ -467,11 +467,11 @@ func (k *IndexDataStoreKey) ToString() string { sb.WriteString(k.IndexID) for _, v := range k.FieldValues { - if v == "" { + if len(v) == 0 { break } sb.WriteByte('/') - sb.WriteString(v) + sb.WriteString(string(v)) } return sb.String() @@ -489,7 +489,7 @@ func (k IndexDataStoreKey) Equal(other IndexDataStoreKey) bool { return false } for i := range k.FieldValues { - if k.FieldValues[i] != other.FieldValues[i] { + if string(k.FieldValues[i]) != string(other.FieldValues[i]) { return false } } diff --git a/core/key_test.go b/core/key_test.go index d0b0fb104f..2e023449df 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -157,6 +157,14 @@ func TestNewIndexKeyFromString_IfFullKeyString_ReturnKey(t *testing.T) { assert.Equal(t, key.IndexName, "idx") } +func toFieldValues(values ...string) [][]byte { + var result [][]byte = make([][]byte, 0, len(values)) + for _, value := range values { + result = append(result, []byte(value)) + } + return result +} + func TestIndexDatastoreKey_ToString(t *testing.T) { cases := []struct { Key IndexDataStoreKey @@ -183,7 +191,7 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { Key: IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"3"}, + FieldValues: toFieldValues("3"), }, Expected: "/1/2/3", }, @@ -191,27 +199,27 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { Key: IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, Expected: "/1/2/3/4", }, { Key: IndexDataStoreKey{ CollectionID: "1", - FieldValues: []string{"3"}, + FieldValues: toFieldValues("3"), }, Expected: "/1", }, { Key: IndexDataStoreKey{ IndexID: "2", - FieldValues: []string{"3"}, + FieldValues: toFieldValues("3"), }, Expected: "", }, { Key: IndexDataStoreKey{ - FieldValues: []string{"3"}, + FieldValues: toFieldValues("3"), }, Expected: "", }, @@ -219,7 +227,7 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { Key: IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"", ""}, + FieldValues: toFieldValues("", ""), }, Expected: "/1/2", }, @@ -227,7 +235,7 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { Key: IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"", "3"}, + FieldValues: toFieldValues("", "3"), }, Expected: "/1/2", }, @@ -235,7 +243,7 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { Key: IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "", "4"}, + FieldValues: toFieldValues("3", "", "4"), }, Expected: "/1/2/3", }, @@ -249,7 +257,7 @@ func TestIndexDatastoreKey_Bytes(t *testing.T) { key := IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), } assert.Equal(t, key.Bytes(), []byte("/1/2/3/4")) } @@ -258,7 +266,7 @@ func TestIndexDatastoreKey_ToDS(t *testing.T) { key := IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), } assert.Equal(t, key.ToDS(), ds.NewKey("/1/2/3/4")) } @@ -269,22 +277,22 @@ func TestIndexDatastoreKey_EqualTrue(t *testing.T) { { CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, { CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, }, { { CollectionID: "1", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, { CollectionID: "1", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, }, { @@ -334,34 +342,34 @@ func TestIndexDatastoreKey_EqualFalse(t *testing.T) { { CollectionID: "1", IndexID: "2", - FieldValues: []string{"4", "3"}, + FieldValues: toFieldValues("4", "3"), }, { CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, }, { { CollectionID: "1", IndexID: "2", - FieldValues: []string{"3"}, + FieldValues: toFieldValues("3"), }, { CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, }, { { CollectionID: "1", - FieldValues: []string{"3", "", "4"}, + FieldValues: toFieldValues("3", "", "4"), }, { CollectionID: "1", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }, }, } @@ -377,7 +385,7 @@ func TestNewIndexDataStoreKey_ValidKey(t *testing.T) { assert.Equal(t, str, IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"3"}, + FieldValues: toFieldValues("3"), }) str, err = NewIndexDataStoreKey("/1/2/3/4") @@ -385,7 +393,7 @@ func TestNewIndexDataStoreKey_ValidKey(t *testing.T) { assert.Equal(t, str, IndexDataStoreKey{ CollectionID: "1", IndexID: "2", - FieldValues: []string{"3", "4"}, + FieldValues: toFieldValues("3", "4"), }) } diff --git a/db/index.go b/db/index.go index 76a713a129..b3e6a27f20 100644 --- a/db/index.go +++ b/db/index.go @@ -126,20 +126,21 @@ func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataS } } - storeValue := "" + var storeValue []byte if isNil { - storeValue = indexFieldNilValue + storeValue = []byte(indexFieldNilValue) } else { data, err := i.convertFunc(fieldVal) if err != nil { return core.IndexDataStoreKey{}, NewErrCanNotIndexInvalidFieldValue(err) } - storeValue = indexFieldValuePrefix + string(data) + storeValue = []byte(string(indexFieldValuePrefix) + string(data)) } indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) - indexDataStoreKey.FieldValues = []string{storeValue, indexFieldValuePrefix + doc.Key().String()} + indexDataStoreKey.FieldValues = [][]byte{storeValue, + []byte(string(indexFieldValuePrefix) + doc.Key().String())} return indexDataStoreKey, nil } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index e1aaf12a10..aa3f027ae1 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -82,7 +82,7 @@ type indexKeyBuilder struct { colName string fieldName string doc *client.Document - values []string + values [][]byte isUnique bool } @@ -114,7 +114,7 @@ func (b *indexKeyBuilder) Doc(doc *client.Document) *indexKeyBuilder { // Values sets the values for the index key. // It will override the field values stored in the document. -func (b *indexKeyBuilder) Values(values ...string) *indexKeyBuilder { +func (b *indexKeyBuilder) Values(values ...[]byte) *indexKeyBuilder { b.values = values return b } @@ -159,16 +159,17 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { } if b.doc != nil { - var fieldStrVal string + var fieldBytesVal []byte if len(b.values) == 0 { fieldVal, err := b.doc.Get(b.fieldName) require.NoError(b.f.t, err) - fieldStrVal = fmt.Sprintf("%s%v", testValuePrefix, fieldVal) + fieldBytesVal = []byte(fmt.Sprintf("%s%v", testValuePrefix, fieldVal)) } else { - fieldStrVal = b.values[0] + fieldBytesVal = b.values[0] } - key.FieldValues = []string{fieldStrVal, testValuePrefix + b.doc.Key().String()} + key.FieldValues = [][]byte{[]byte(fieldBytesVal), + []byte(testValuePrefix + b.doc.Key().String())} } else if len(b.values) > 0 { key.FieldValues = b.values } @@ -479,7 +480,7 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { keyBuilder := newIndexKeyBuilder(f).Col(collection.Name()).Field("field").Doc(doc) if tc.Stored != "" { - keyBuilder.Values(testValuePrefix + tc.Stored) + keyBuilder.Values([]byte(testValuePrefix + tc.Stored)) } key := keyBuilder.Build() @@ -506,7 +507,7 @@ func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { f.saveDocToCollection(doc, f.users) key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). - Values(testNilValue).Build() + Values([]byte(testNilValue)).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -892,7 +893,7 @@ func TestNonUniqueUpdate_IfFailsToUpdateIndex_ReturnError(t *testing.T) { validKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() invalidKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc). - Values("invalid").Build() + Values([]byte("invalid")).Build() err := f.txn.Datastore().Delete(f.ctx, validKey.ToDS()) require.NoError(f.t, err) @@ -1004,7 +1005,7 @@ func TestNonUpdate_IfIndexedFieldWasNil_ShouldDeleteIt(t *testing.T) { f.saveDocToCollection(doc, f.users) oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). - Values(testNilValue).Build() + Values([]byte(testNilValue)).Build() err = doc.Set(usersNameFieldName, "John") require.NoError(f.t, err) From a16ff589aa6a459fd1dc5431bd91593329732624 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 12 Jun 2023 21:44:25 +0200 Subject: [PATCH 092/120] Format --- client/collection.go | 2 +- client/index.go | 4 ++-- core/key.go | 2 +- db/indexed_docs_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/collection.go b/client/collection.go index 75d737d2cb..9c91dccb7c 100644 --- a/client/collection.go +++ b/client/collection.go @@ -139,7 +139,7 @@ type Collection interface { // CreateIndex creates a new index on the collection. // `IndexDescription` contains the description of the index to be created. - // `IndexDescription.Name` must start with a letter or an underscore and can + // `IndexDescription.Name` must start with a letter or an underscore and can // only contain letters, numbers, and underscores. // If the name of the index is not provided, it will be generated. CreateIndex(context.Context, IndexDescription) (IndexDescription, error) diff --git a/client/index.go b/client/index.go index 9941f8877f..b916875470 100644 --- a/client/index.go +++ b/client/index.go @@ -15,7 +15,7 @@ type IndexDirection string const ( // Ascending is the value to use for an ascending fields - Ascending IndexDirection = "ASC" + Ascending IndexDirection = "ASC" // Descending is the value to use for an descending fields Descending IndexDirection = "DESC" ) @@ -23,7 +23,7 @@ const ( // IndexFieldDescription describes how a field is being indexed. type IndexedFieldDescription struct { // Name contains the name of the field. - Name string + Name string // Direction contains the direction of the index. Direction IndexDirection } diff --git a/core/key.go b/core/key.go index ad49fb84a3..0f95dbe506 100644 --- a/core/key.go +++ b/core/key.go @@ -477,7 +477,7 @@ func (k *IndexDataStoreKey) ToString() string { return sb.String() } -// Equal returns true if the two keys are equal +// Equal returns true if the two keys are equal func (k IndexDataStoreKey) Equal(other IndexDataStoreKey) bool { if k.CollectionID != other.CollectionID { return false diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index aa3f027ae1..03d8dbd799 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -168,7 +168,7 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { fieldBytesVal = b.values[0] } - key.FieldValues = [][]byte{[]byte(fieldBytesVal), + key.FieldValues = [][]byte{[]byte(fieldBytesVal), []byte(testValuePrefix + b.doc.Key().String())} } else if len(b.values) > 0 { key.FieldValues = b.values From 5b952880dd0badb9db52f1fefe3ebbc68674a0e2 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 12 Jun 2023 22:35:29 +0200 Subject: [PATCH 093/120] Switch IndexDataStoreKey members to uint32 --- core/key.go | 24 +++++----- core/key_test.go | 98 ++++++++++++++++++++--------------------- db/index.go | 11 ++--- db/indexed_docs_test.go | 4 +- 4 files changed, 69 insertions(+), 68 deletions(-) diff --git a/core/key.go b/core/key.go index 0f95dbe506..50e1400ba9 100644 --- a/core/key.go +++ b/core/key.go @@ -70,10 +70,10 @@ var _ Key = (*DataStoreKey)(nil) // IndexDataStoreKey is key of an indexed document in the database. type IndexDataStoreKey struct { - // CollectionID is the id (unique number) of the collection - CollectionID string - // IndexID is the id (unique number) of the index - IndexID string + // CollectionID is the id of the collection + CollectionID uint32 + // IndexID is the id of the index + IndexID uint32 // FieldValues is the values of the fields in the index FieldValues [][]byte } @@ -412,18 +412,18 @@ func NewIndexDataStoreKey(key string) (IndexDataStoreKey, error) { return IndexDataStoreKey{}, ErrInvalidKey } - _, err := strconv.Atoi(elements[0]) + colID, err := strconv.Atoi(elements[0]) if err != nil { return IndexDataStoreKey{}, ErrInvalidKey } - indexKey := IndexDataStoreKey{CollectionID: elements[0]} + indexKey := IndexDataStoreKey{CollectionID: uint32(colID)} - _, err = strconv.Atoi(elements[1]) + indID, err := strconv.Atoi(elements[1]) if err != nil { return IndexDataStoreKey{}, ErrInvalidKey } - indexKey.IndexID = elements[1] + indexKey.IndexID = uint32(indID) for i := 2; i < len(elements); i++ { _, err = strconv.Atoi(elements[i]) @@ -454,17 +454,17 @@ func (k *IndexDataStoreKey) ToDS() ds.Key { func (k *IndexDataStoreKey) ToString() string { sb := strings.Builder{} - if k.CollectionID == "" { + if k.CollectionID == 0 { return "" } sb.WriteByte('/') - sb.WriteString(k.CollectionID) + sb.WriteString(strconv.Itoa(int(k.CollectionID))) - if k.IndexID == "" { + if k.IndexID == 0 { return sb.String() } sb.WriteByte('/') - sb.WriteString(k.IndexID) + sb.WriteString(strconv.Itoa(int(k.IndexID))) for _, v := range k.FieldValues { if len(v) == 0 { diff --git a/core/key_test.go b/core/key_test.go index 2e023449df..39b3443fd2 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -176,43 +176,43 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { }, { Key: IndexDataStoreKey{ - CollectionID: "1", + CollectionID: 1, }, Expected: "/1", }, { Key: IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, }, Expected: "/1/2", }, { Key: IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3"), }, Expected: "/1/2/3", }, { Key: IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), }, Expected: "/1/2/3/4", }, { Key: IndexDataStoreKey{ - CollectionID: "1", + CollectionID: 1, FieldValues: toFieldValues("3"), }, Expected: "/1", }, { Key: IndexDataStoreKey{ - IndexID: "2", + IndexID: 2, FieldValues: toFieldValues("3"), }, Expected: "", @@ -225,24 +225,24 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { }, { Key: IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("", ""), }, Expected: "/1/2", }, { Key: IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("", "3"), }, Expected: "/1/2", }, { Key: IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "", "4"), }, Expected: "/1/2/3", @@ -255,8 +255,8 @@ func TestIndexDatastoreKey_ToString(t *testing.T) { func TestIndexDatastoreKey_Bytes(t *testing.T) { key := IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), } assert.Equal(t, key.Bytes(), []byte("/1/2/3/4")) @@ -264,8 +264,8 @@ func TestIndexDatastoreKey_Bytes(t *testing.T) { func TestIndexDatastoreKey_ToDS(t *testing.T) { key := IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), } assert.Equal(t, key.ToDS(), ds.NewKey("/1/2/3/4")) @@ -275,32 +275,32 @@ func TestIndexDatastoreKey_EqualTrue(t *testing.T) { cases := [][]IndexDataStoreKey{ { { - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), }, { - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), }, }, { { - CollectionID: "1", + CollectionID: 1, FieldValues: toFieldValues("3", "4"), }, { - CollectionID: "1", + CollectionID: 1, FieldValues: toFieldValues("3", "4"), }, }, { { - CollectionID: "1", + CollectionID: 1, }, { - CollectionID: "1", + CollectionID: 1, }, }, } @@ -314,61 +314,61 @@ func TestIndexDatastoreKey_EqualFalse(t *testing.T) { cases := [][]IndexDataStoreKey{ { { - CollectionID: "1", + CollectionID: 1, }, { - CollectionID: "2", + CollectionID: 2, }, }, { { - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, }, { - CollectionID: "1", - IndexID: "3", + CollectionID: 1, + IndexID: 3, }, }, { { - CollectionID: "1", + CollectionID: 1, }, { - IndexID: "1", + IndexID: 1, }, }, { { - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("4", "3"), }, { - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), }, }, { { - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3"), }, { - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), }, }, { { - CollectionID: "1", + CollectionID: 1, FieldValues: toFieldValues("3", "", "4"), }, { - CollectionID: "1", + CollectionID: 1, FieldValues: toFieldValues("3", "4"), }, }, @@ -383,16 +383,16 @@ func TestNewIndexDataStoreKey_ValidKey(t *testing.T) { str, err := NewIndexDataStoreKey("/1/2/3") assert.NoError(t, err) assert.Equal(t, str, IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3"), }) str, err = NewIndexDataStoreKey("/1/2/3/4") assert.NoError(t, err) assert.Equal(t, str, IndexDataStoreKey{ - CollectionID: "1", - IndexID: "2", + CollectionID: 1, + IndexID: 2, FieldValues: toFieldValues("3", "4"), }) } diff --git a/db/index.go b/db/index.go index b3e6a27f20..abb091ee13 100644 --- a/db/index.go +++ b/db/index.go @@ -137,8 +137,8 @@ func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataS storeValue = []byte(string(indexFieldValuePrefix) + string(data)) } indexDataStoreKey := core.IndexDataStoreKey{} - indexDataStoreKey.CollectionID = strconv.Itoa(int(i.collection.ID())) - indexDataStoreKey.IndexID = strconv.Itoa(int(i.desc.ID)) + indexDataStoreKey.CollectionID = i.collection.ID() + indexDataStoreKey.IndexID = i.desc.ID indexDataStoreKey.FieldValues = [][]byte{storeValue, []byte(string(indexFieldValuePrefix) + doc.Key().String())} return indexDataStoreKey, nil @@ -155,7 +155,8 @@ func (i *collectionSimpleIndex) Save( } err = txn.Datastore().Put(ctx, key.ToDS(), []byte{}) if err != nil { - return NewErrFailedToStoreIndexedField(key.IndexID, err) + field, _ := i.collection.Description().GetFieldByID(strconv.Itoa(int(key.IndexID))) + return NewErrFailedToStoreIndexedField(field.Name, err) } return nil } @@ -207,8 +208,8 @@ func iteratePrefixKeys( } func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn) error { prefixKey := core.IndexDataStoreKey{} - prefixKey.CollectionID = strconv.Itoa(int(i.collection.ID())) - prefixKey.IndexID = strconv.Itoa(int(i.desc.ID)) + prefixKey.CollectionID = i.collection.ID() + prefixKey.IndexID = i.desc.ID err := iteratePrefixKeys(ctx, prefixKey.ToString(), txn.Datastore(), func(ctx context.Context, key ds.Key) error { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 03d8dbd799..7952b03f3d 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -143,7 +143,7 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { if collection == nil { panic(errors.New("collection not found")) } - key.CollectionID = strconv.Itoa(int(collection.ID())) + key.CollectionID = collection.ID() if b.fieldName == "" { return key @@ -153,7 +153,7 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { require.NoError(b.f.t, err) for _, index := range indexes { if index.Fields[0].Name == b.fieldName { - key.IndexID = strconv.Itoa(int(index.ID)) + key.IndexID = index.ID break } } From a9662d8e7db2f116d6a4624f7179195444637b4d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 12 Jun 2023 22:37:58 +0200 Subject: [PATCH 094/120] Format --- db/errors.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/errors.go b/db/errors.go index 6c35f6bc73..7c736e34b1 100644 --- a/db/errors.go +++ b/db/errors.go @@ -35,8 +35,8 @@ const ( errInvalidCRDTType string = "only default or LWW (last writer wins) CRDT types are supported" errCannotDeleteField string = "deleting an existing field is not supported" errFieldKindNotFound string = "no type found for given name" - errDocumentAlreadyExists string = "a document with the given dockey already exists" - errDocumentDeleted string = "a document with the given dockey has been deleted" + errDocumentAlreadyExists string = "a document with the given dockey already exists" + errDocumentDeleted string = "a document with the given dockey has been deleted" errIndexMissingFields string = "index missing fields" errNonZeroIndexIDProvided string = "non-zero index ID provided" errIndexFieldMissingName string = "index field missing name" From a2c1c262973dc140be12d3712c319069294822b7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Jun 2023 15:24:14 +0200 Subject: [PATCH 095/120] Small refactor --- db/collection_index.go | 47 ++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index 32c2ff714e..7a0119671c 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -72,36 +72,24 @@ func (db *db) getAllCollectionIndexes( txn datastore.Txn, ) ([]collectionIndexDescription, error) { prefix := core.NewCollectionIndexKey("", "") - q, err := txn.Systemstore().Query(ctx, query.Query{ - Prefix: prefix.ToString(), - }) + + indexMap, err := deserializePrefix[client.IndexDescription](ctx, + prefix.ToString(), txn.Systemstore()) + if err != nil { - return nil, NewErrFailedToCreateCollectionQuery(err) + return nil, err } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() - indexes := make([]collectionIndexDescription, 0) - for res := range q.Next() { - if res.Error != nil { - return nil, res.Error - } + indexes := make([]collectionIndexDescription, 0, len(indexMap)) - var colDesk client.IndexDescription - err = json.Unmarshal(res.Value, &colDesk) - if err != nil { - return nil, NewErrInvalidStoredIndex(err) - } - indexKey, err := core.NewCollectionIndexKeyFromString(res.Key) + for indexKeyStr, index := range indexMap { + indexKey, err := core.NewCollectionIndexKeyFromString(indexKeyStr) if err != nil { return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) } indexes = append(indexes, collectionIndexDescription{ CollectionName: indexKey.CollectionName, - Index: colDesk, + Index: index, }) } @@ -114,7 +102,16 @@ func (db *db) getCollectionIndexes( colName string, ) ([]client.IndexDescription, error) { prefix := core.NewCollectionIndexKey(colName, "") - return deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) + indexMap, err := deserializePrefix[client.IndexDescription](ctx, + prefix.ToString(), txn.Systemstore()) + if err != nil { + return nil, err + } + indexes := make([]client.IndexDescription, 0, len(indexMap)) + for _, index := range indexMap { + indexes = append(indexes, index) + } + return indexes, nil } func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { @@ -496,7 +493,7 @@ func generateIndexName(col client.Collection, fields []client.IndexedFieldDescri return sb.String() } -func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Read) ([]T, error) { +func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Read) (map[string]T, error) { q, err := storage.Query(ctx, query.Query{Prefix: prefix}) if err != nil { return nil, NewErrFailedToCreateCollectionQuery(err) @@ -507,7 +504,7 @@ func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Rea } }() - elements := make([]T, 0) + elements := make(map[string]T) for res := range q.Next() { if res.Error != nil { return nil, res.Error @@ -518,7 +515,7 @@ func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Rea if err != nil { return nil, NewErrInvalidStoredIndex(err) } - elements = append(elements, element) + elements[res.Key] = element } return elements, nil } From fafc54a41eefe8890366533d0532dd2c30b52257 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Jun 2023 15:50:55 +0200 Subject: [PATCH 096/120] Make q.Close() be called directly instead of defer --- db/collection_index.go | 10 +++++----- db/index_test.go | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index 7a0119671c..efcc5d7f2f 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -498,24 +498,24 @@ func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Rea if err != nil { return nil, NewErrFailedToCreateCollectionQuery(err) } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() elements := make(map[string]T) for res := range q.Next() { if res.Error != nil { + _ = q.Close() return nil, res.Error } var element T err = json.Unmarshal(res.Value, &element) if err != nil { + _ = q.Close() return nil, NewErrInvalidStoredIndex(err) } elements[res.Key] = element } + if err := q.Close(); err != nil { + return nil, err + } return elements, nil } diff --git a/db/index_test.go b/db/index_test.go index 53a6f3f03f..596917ba3b 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -726,13 +726,30 @@ func TestGetIndexes_IfSystemStoreQueryIteratorFails_ReturnError(t *testing.T) { mockedTxn := f.mockTxn() mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Unset() - mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything). - Return(mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}), nil) + q := mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}) + q.EXPECT().Close().Unset() + q.EXPECT().Close().Return(nil) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Return(q, nil) _, err := f.getAllIndexes() assert.ErrorIs(t, err, testErr) } +func TestGetIndexes_IfSystemStoreHasInvalidData_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Unset() + q := mocks.NewQueryResultsWithValues(t, []byte("invalid")) + q.EXPECT().Close().Unset() + q.EXPECT().Close().Return(nil) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Return(q, nil) + + _, err := f.getAllIndexes() + assert.ErrorIs(t, err, NewErrInvalidStoredIndex(nil)) +} + func TestGetIndexes_IfFailsToReadSeqNumber_ReturnError(t *testing.T) { testErr := errors.New("test error") From f0a79f04ea513112ec80677eee00219dcf1c7f7f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Jun 2023 18:55:22 +0200 Subject: [PATCH 097/120] Remove deferred call to Query.Close() --- db/index.go | 10 ++--- db/indexed_docs_test.go | 99 +++++++++++++++++++++++------------------ 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/db/index.go b/db/index.go index abb091ee13..b3f80b0bd0 100644 --- a/db/index.go +++ b/db/index.go @@ -188,21 +188,21 @@ func iteratePrefixKeys( if err != nil { return err } - defer func() { - if err := q.Close(); err != nil { - log.ErrorE(ctx, "Failed to close collection query", err) - } - }() for res := range q.Next() { if res.Error != nil { + _ = q.Close() return res.Error } err = execFunc(ctx, ds.NewKey(res.Key)) if err != nil { + _ = q.Close() return err } } + if err = q.Close(); err != nil { + return err + } return nil } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 7952b03f3d..cc2e4596c5 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -664,38 +664,66 @@ func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { assert.Len(t, f.getPrefixFromDataStore(prodCatKey.ToString()), 1) } -func TestNonUniqueDrop_IfFailsToQueryDataStorage_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) - f.createUserCollectionIndexOnName() - +func TestNonUniqueDrop_IfDataStorageFails_ReturnError(t *testing.T) { testErr := errors.New("test error") - mockedTxn := f.mockTxn() - mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) - mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything).Unset() - mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything).Return(nil, testErr) - mockedTxn.EXPECT().Datastore().Unset() - mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) - - err := f.dropIndex(usersColName, testUsersColIndexName) - require.ErrorIs(t, err, testErr) -} - -func TestNonUniqueDrop_IfQueryIteratorFails_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) - f.createUserCollectionIndexOnName() + testCases := []struct { + description string + prepareSystemStorage func(*mocks.DSReaderWriter_Expecter) + }{ + { + description: "Fails to query data storage", + prepareSystemStorage: func(mockedDS *mocks.DSReaderWriter_Expecter) { + mockedDS.Query(mock.Anything, mock.Anything).Unset() + mockedDS.Query(mock.Anything, mock.Anything).Return(nil, testErr) + }, + }, + { + description: "Fails to iterate data storage", + prepareSystemStorage: func(mockedDS *mocks.DSReaderWriter_Expecter) { + mockedDS.Query(mock.Anything, mock.Anything).Unset() + q := mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}) + mockedDS.Query(mock.Anything, mock.Anything).Return(q, nil) + q.EXPECT().Close().Unset() + q.EXPECT().Close().Return(nil) + }, + }, + { + description: "Fails to delete from data storage", + prepareSystemStorage: func(mockedDS *mocks.DSReaderWriter_Expecter) { + q := mocks.NewQueryResultsWithResults(t, query.Result{Entry: query.Entry{Key: ""}}) + q.EXPECT().Close().Unset() + q.EXPECT().Close().Return(nil) + mockedDS.Query(mock.Anything, mock.Anything).Return(q, nil) + mockedDS.Delete(mock.Anything, mock.Anything).Unset() + mockedDS.Delete(mock.Anything, mock.Anything).Return(testErr) + }, + }, + { + description: "Fails to close data storage query iterator", + prepareSystemStorage: func(mockedDS *mocks.DSReaderWriter_Expecter) { + q := mocks.NewQueryResultsWithResults(t, query.Result{Entry: query.Entry{Key: ""}}) + q.EXPECT().Close().Unset() + q.EXPECT().Close().Return(testErr) + mockedDS.Query(mock.Anything, mock.Anything).Return(q, nil) + mockedDS.Delete(mock.Anything, mock.Anything).Return(nil) + }, + }, + } - testErr := errors.New("test error") + for _, tc := range testCases { + f := newIndexTestFixture(t) + f.createUserCollectionIndexOnName() - mockedTxn := f.mockTxn() - mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) - mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything). - Return(mocks.NewQueryResultsWithResults(t, query.Result{Error: testErr}), nil) - mockedTxn.EXPECT().Datastore().Unset() - mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) + mockedTxn := f.mockTxn() + mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) + tc.prepareSystemStorage(mockedTxn.MockDatastore.EXPECT()) + mockedTxn.EXPECT().Datastore().Unset() + mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) - err := f.dropIndex(usersColName, testUsersColIndexName) - require.ErrorIs(t, err, testErr) + err := f.dropIndex(usersColName, testUsersColIndexName) + require.ErrorIs(t, err, testErr, tc.description) + } } func TestNonUniqueDrop_ShouldCloseQueryIterator(t *testing.T) { @@ -717,23 +745,6 @@ func TestNonUniqueDrop_ShouldCloseQueryIterator(t *testing.T) { assert.NoError(t, err) } -func TestNonUniqueDrop_IfDatastoreFailsToDelete_ReturnError(t *testing.T) { - f := newIndexTestFixture(t) - f.createUserCollectionIndexOnName() - - mockedTxn := f.mockTxn() - mockedTxn.MockDatastore = mocks.NewDSReaderWriter(t) - mockedTxn.MockDatastore.EXPECT().Query(mock.Anything, mock.Anything). - Return(mocks.NewQueryResultsWithValues(t, []byte{}), nil) - mockedTxn.MockDatastore.EXPECT().Delete(mock.Anything, mock.Anything). - Return(errors.New("error")) - mockedTxn.EXPECT().Datastore().Unset() - mockedTxn.EXPECT().Datastore().Return(mockedTxn.MockDatastore) - - err := f.dropIndex(usersColName, testUsersColIndexName) - require.ErrorIs(t, err, NewCanNotDeleteIndexedField(nil)) -} - func TestNonUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() From 52076d95154fe06e0a5e79ad9132a76bd10a2366 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Jun 2023 12:07:14 +0200 Subject: [PATCH 098/120] Return UnsupportedType error instead of panicking --- db/collection_index.go | 19 ++++++++--- db/errors.go | 36 +++++++++++++++---- db/index.go | 31 +++++++++-------- db/index_test.go | 76 +++++++++++++++++++++++++++++++++++++++++ db/indexed_docs_test.go | 15 ++++---- 5 files changed, 144 insertions(+), 33 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index efcc5d7f2f..2780b80fd7 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -128,6 +128,7 @@ func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *cl return nil } +// collectIndexedFields returns all fields that are indexed by all collection indexes. func (c *collection) collectIndexedFields() []*client.FieldDescription { fieldsMap := make(map[string]*client.FieldDescription) for _, index := range c.indexes { @@ -320,13 +321,18 @@ func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]Colle } prefix := core.NewCollectionIndexKey(c.Name(), "") - indexes, err := deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) + indexDescriptions, err := deserializePrefix[client.IndexDescription]( + ctx, prefix.ToString(), txn.Systemstore()) if err != nil { return nil, err } - colIndexes := make([]CollectionIndex, 0, len(indexes)) - for _, index := range indexes { - colIndexes = append(colIndexes, NewCollectionIndex(c, index)) + colIndexes := make([]CollectionIndex, 0, len(indexDescriptions)) + for _, desc := range indexDescriptions { + index, err := NewCollectionIndex(c, desc) + if err != nil { + return nil, err + } + colIndexes = append(colIndexes, index) } descriptions := make([]client.IndexDescription, 0, len(colIndexes)) @@ -398,7 +404,10 @@ func (c *collection) createIndex( if err != nil { return nil, err } - colIndex := NewCollectionIndex(c, desc) + colIndex, err := NewCollectionIndex(c, desc) + if err != nil { + return nil, err + } c.desc.Indexes = append(c.desc.Indexes, colIndex.Description()) return colIndex, nil } diff --git a/db/errors.go b/db/errors.go index 7c736e34b1..1fe22716d9 100644 --- a/db/errors.go +++ b/db/errors.go @@ -49,12 +49,13 @@ const ( errCollectionDoesntExisting string = "collection with given name doesn't exist" errFailedToStoreIndexedField string = "failed to store indexed field" errFailedToReadStoredIndexDesc string = "failed to read stored index description" - errCanNotIndexInvalidFieldValue string = "can not index invalid field value" errCanNotDeleteIndexedField string = "can not delete indexed field" errCanNotAddIndexWithPatch string = "adding indexes via patch is not supported" errCanNotDropIndexWithPatch string = "dropping indexes via patch is not supported" errCanNotChangeIndexWithPatch string = "changing indexes via patch is not supported" errIndexWithNameDoesNotExists string = "index with name doesn't exists" + errInvalidFieldValue string = "invalid field value" + errUnsupportedIndexFieldType string = "unsupported index field type" ) var ( @@ -155,12 +156,6 @@ func NewErrFailedToReadStoredIndexDesc(inner error) error { return errors.Wrap(errFailedToReadStoredIndexDesc, inner) } -// NewErrCanNotIndexInvalidFieldValue returns a new error indicating that the field value is invalid -// and cannot be indexed. -func NewErrCanNotIndexInvalidFieldValue(inner error) error { - return errors.Wrap(errCanNotIndexInvalidFieldValue, inner) -} - // NewCanNotDeleteIndexedField returns a new error a failed attempt to delete an indexed field func NewCanNotDeleteIndexedField(inner error) error { return errors.Wrap(errCanNotDeleteIndexedField, inner) @@ -304,6 +299,8 @@ func NewErrDocumentDeleted(dockey string) error { ) } +// NewErrIndexWithNameAlreadyExists returns a new error indicating that an index with the +// given name already exists. func NewErrIndexWithNameAlreadyExists(indexName string) error { return errors.New( errIndexWithNameAlreadyExists, @@ -311,6 +308,8 @@ func NewErrIndexWithNameAlreadyExists(indexName string) error { ) } +// NewErrIndexWithNameDoesNotExists returns a new error indicating that an index with the +// given name does not exist. func NewErrIndexWithNameDoesNotExists(indexName string) error { return errors.New( errIndexWithNameDoesNotExists, @@ -318,6 +317,8 @@ func NewErrIndexWithNameDoesNotExists(indexName string) error { ) } +// NewErrCannotAddIndexWithPatch returns a new error indicating that an index cannot be added +// with a patch. func NewErrCannotAddIndexWithPatch(proposedName string) error { return errors.New( errCanNotAddIndexWithPatch, @@ -325,9 +326,30 @@ func NewErrCannotAddIndexWithPatch(proposedName string) error { ) } +// NewErrCannotDropIndexWithPatch returns a new error indicating that an index cannot be dropped +// with a patch. func NewErrCannotDropIndexWithPatch(indexName string) error { return errors.New( errCanNotDropIndexWithPatch, errors.NewKV("Name", indexName), ) } + +// NewErrInvalidFieldValue returns a new error indicating that the given value is invalid for the +// given field kind. +func NewErrInvalidFieldValue(kind client.FieldKind, value any) error { + return errors.New( + errInvalidFieldValue, + errors.NewKV("Kind", kind), + errors.NewKV("Value", value), + ) +} + +// NewErrUnsupportedIndexFieldType returns a new error indicating that the given field kind is not +// supported for indexing. +func NewErrUnsupportedIndexFieldType(kind client.FieldKind) error { + return errors.New( + errUnsupportedIndexFieldType, + errors.NewKV("Kind", kind), + ) +} diff --git a/db/index.go b/db/index.go index b3f80b0bd0..c664b4a4e2 100644 --- a/db/index.go +++ b/db/index.go @@ -46,51 +46,51 @@ type CollectionIndex interface { Description() client.IndexDescription } -func getFieldValConverter(kind client.FieldKind) func(any) ([]byte, error) { +func getFieldValConverter(kind client.FieldKind) (func(any) ([]byte, error), error) { switch kind { case client.FieldKind_STRING: return func(val any) ([]byte, error) { return []byte(val.(string)), nil - } + }, nil case client.FieldKind_INT: return func(val any) ([]byte, error) { intVal, ok := val.(int64) if !ok { - return nil, errors.New("invalid int value") + return nil, NewErrInvalidFieldValue(kind, val) } return []byte(strconv.FormatInt(intVal, 10)), nil - } + }, nil case client.FieldKind_FLOAT: return func(val any) ([]byte, error) { floatVal, ok := val.(float64) if !ok { - return nil, errors.New("invalid float value") + return nil, NewErrInvalidFieldValue(kind, val) } return []byte(strconv.FormatFloat(floatVal, 'f', -1, 64)), nil - } + }, nil case client.FieldKind_BOOL: return func(val any) ([]byte, error) { boolVal, ok := val.(bool) if !ok { - return nil, errors.New("invalid bool value") + return nil, NewErrInvalidFieldValue(kind, val) } var intVal int64 = 0 if boolVal { intVal = 1 } return []byte(strconv.FormatInt(intVal, 10)), nil - } + }, nil case client.FieldKind_DATETIME: return func(val any) ([]byte, error) { timeStrVal := val.(string) _, err := time.Parse(time.RFC3339, timeStrVal) if err != nil { - return nil, err + return nil, NewErrInvalidFieldValue(kind, val) } return []byte(timeStrVal), nil - } + }, nil default: - panic("there is no test for this case") + return nil, NewErrUnsupportedIndexFieldType(kind) } } @@ -98,13 +98,14 @@ func getFieldValConverter(kind client.FieldKind) func(any) ([]byte, error) { func NewCollectionIndex( collection client.Collection, desc client.IndexDescription, -) CollectionIndex { +) (CollectionIndex, error) { index := &collectionSimpleIndex{collection: collection, desc: desc} schema := collection.Description().Schema fieldID := schema.GetFieldKey(desc.Fields[0].Name) field := schema.Fields[fieldID] - index.convertFunc = getFieldValConverter(field.Kind) - return index + var e error + index.convertFunc, e = getFieldValConverter(field.Kind) + return index, e } type collectionSimpleIndex struct { @@ -132,7 +133,7 @@ func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataS } else { data, err := i.convertFunc(fieldVal) if err != nil { - return core.IndexDataStoreKey{}, NewErrCanNotIndexInvalidFieldValue(err) + return core.IndexDataStoreKey{}, err } storeValue = []byte(string(indexFieldValuePrefix) + string(data)) } diff --git a/db/index_test.go b/db/index_test.go index 596917ba3b..c1bd8f6a3c 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -601,6 +601,41 @@ func TestCreateIndex_NewCollectionDescription_ShouldIncludeIndexDescription(t *t require.NotEmpty(t, col.Description().Indexes[1].Name) } +func TestCreateIndex_IfAttemptToIndexOnUnsupportedType_ReturnError(t *testing.T) { + f := newIndexTestFixtureBare(t) + + const unsupportedKind = client.FieldKind_BOOL_ARRAY + + desc := client.CollectionDescription{ + Name: "testTypeCol", + Schema: client.SchemaDescription{ + Fields: []client.FieldDescription{ + { + Name: "_key", + Kind: client.FieldKind_DocKey, + }, + { + Name: "field", + Kind: unsupportedKind, + Typ: client.LWW_REGISTER, + }, + }, + }, + } + + collection := f.createCollection(desc) + + indexDesc := client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + {Name: "field", Direction: client.Ascending}, + }, + } + + _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + require.ErrorIs(f.t, err, NewErrUnsupportedIndexFieldType(unsupportedKind)) + f.commitTxn() +} + func TestCreateIndex_IfFailedToReadIndexUponRetrievingCollectionDesc_ReturnError(t *testing.T) { f := newIndexTestFixture(t) @@ -1008,6 +1043,47 @@ func TestCollectionGetIndexes_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { assert.Equal(t, testUsersColIndexName, indexes[0].Name) } +func TestCollectionGetIndexes_IfStoredIndexWithUnsupportedType_ReturnError(t *testing.T) { + f := newIndexTestFixtureBare(t) + + const unsupportedKind = client.FieldKind_BOOL_ARRAY + + desc := client.CollectionDescription{ + Name: "testTypeCol", + Schema: client.SchemaDescription{ + Fields: []client.FieldDescription{ + { + Name: "_key", + Kind: client.FieldKind_DocKey, + }, + { + Name: "field", + Kind: unsupportedKind, + Typ: client.LWW_REGISTER, + }, + }, + }, + } + + collection := f.createCollection(desc) + + indexDesc := client.IndexDescription{ + Fields: []client.IndexedFieldDescription{ + {Name: "field", Direction: client.Ascending}, + }, + } + indexDescData, err := json.Marshal(indexDesc) + require.NoError(t, err) + + mockedTxn := f.mockTxn() + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Unset() + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything). + Return(mocks.NewQueryResultsWithValues(t, indexDescData), nil) + + _, err = collection.WithTxn(mockedTxn).GetIndexes(f.ctx) + require.ErrorIs(t, err, NewErrUnsupportedIndexFieldType(unsupportedKind)) +} + func TestCollectionGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { f := newIndexTestFixture(t) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index cc2e4596c5..63696690b7 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -212,7 +212,7 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E indexOnNameDescData, err := json.Marshal(desc) require.NoError(f.t, err) - colIndexKey := core.NewCollectionIndexKey(f.users.Description().Name, "") + colIndexKey := core.NewCollectionIndexKey(usersColName, "") matchPrefixFunc := func(q query.Query) bool { return q.Prefix == colIndexKey.ToDS().String() } @@ -226,7 +226,7 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E systemStoreOn.Query(mock.Anything, mock.Anything).Maybe(). Return(mocks.NewQueryResultsWithValues(f.t), nil) - colKey := core.NewCollectionKey(f.users.Name()) + colKey := core.NewCollectionKey(usersColName) systemStoreOn.Get(mock.Anything, colKey.ToDS()).Maybe().Return([]byte(userColVersionID), nil) colVersionIDKey := core.NewCollectionSchemaVersionKey(userColVersionID) @@ -239,11 +239,13 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E require.NoError(f.t, err) systemStoreOn.Get(mock.Anything, colVersionIDKey.ToDS()).Maybe().Return(colDescBytes, nil) - colIndexOnNameKey := core.NewCollectionIndexKey(f.users.Description().Name, testUsersColIndexName) + colIndexOnNameKey := core.NewCollectionIndexKey(usersColName, testUsersColIndexName) systemStoreOn.Get(mock.Anything, colIndexOnNameKey.ToDS()).Maybe().Return(indexOnNameDescData, nil) - sequenceKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, f.users.ID())) - systemStoreOn.Get(mock.Anything, sequenceKey.ToDS()).Maybe().Return([]byte{0, 0, 0, 0, 0, 0, 0, 1}, nil) + if f.users != nil { + sequenceKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, f.users.ID())) + systemStoreOn.Get(mock.Anything, sequenceKey.ToDS()).Maybe().Return([]byte{0, 0, 0, 0, 0, 0, 0, 1}, nil) + } systemStoreOn.Get(mock.Anything, mock.Anything).Maybe().Return([]byte{}, nil) @@ -473,7 +475,8 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { err = collection.Create(f.ctx, doc) f.commitTxn() if tc.ShouldFail { - require.ErrorIs(f.t, err, NewErrCanNotIndexInvalidFieldValue(nil), "test case: %s", tc.Name) + require.ErrorIs(f.t, err, + NewErrInvalidFieldValue(tc.FieldKind, tc.FieldVal), "test case: %s", tc.Name) } else { assertMsg := fmt.Sprintf("test case: %s", tc.Name) require.NoError(f.t, err, assertMsg) From 85e69fb0fc3df1afd9aac19da6b423ef40f23c52 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Jun 2023 12:42:22 +0200 Subject: [PATCH 099/120] Correctly read FieldID --- client/descriptions.go | 4 ++-- db/collection.go | 6 +++++- db/errors.go | 38 +++++++++++++++++++++++++++++++------- db/index.go | 13 +++++++++---- db/index_test.go | 16 ++++++++++++++++ 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/client/descriptions.go b/client/descriptions.go index 8570cb6201..6e4f30b1bb 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -53,10 +53,10 @@ func (col CollectionDescription) GetField(name string) (FieldDescription, bool) // GetFieldByID searches for a field with the given ID. If such a field is found it // will return it and true, if it is not found it will return false. -func (col CollectionDescription) GetFieldByID(id string) (FieldDescription, bool) { +func (col CollectionDescription) GetFieldByID(id FieldID) (FieldDescription, bool) { if !col.Schema.IsEmpty() { for _, field := range col.Schema.Fields { - if field.ID.String() == id { + if field.ID == id { return field, true } } diff --git a/db/collection.go b/db/collection.go index f424e000da..a4215cee33 100644 --- a/db/collection.go +++ b/db/collection.go @@ -1040,7 +1040,11 @@ func (c *collection) saveValueToMerkleCRDT( args ...any) (ipld.Node, uint64, error) { switch ctype { case client.LWW_REGISTER: - field, _ := c.Description().GetFieldByID(key.FieldId) + fieldID, err := strconv.Atoi(key.FieldId) + if err != nil { + return nil, 0, err + } + field, _ := c.Description().GetFieldByID(client.FieldID(fieldID)) merkleCRDT, err := c.db.crdtFactory.InstanceWithStores( txn, core.NewCollectionSchemaVersionKey(c.Schema().VersionID), diff --git a/db/errors.go b/db/errors.go index 1fe22716d9..3a92bc79f1 100644 --- a/db/errors.go +++ b/db/errors.go @@ -56,6 +56,8 @@ const ( errIndexWithNameDoesNotExists string = "index with name doesn't exists" errInvalidFieldValue string = "invalid field value" errUnsupportedIndexFieldType string = "unsupported index field type" + errIndexDescriptionHasNoFields string = "index description has no fields" + errIndexDescHasNonExistingField string = "index description has non existing field" ) var ( @@ -145,13 +147,14 @@ func NewErrCanNotReadCollection(colName string, inner error) error { return errors.Wrap(errCollectionDoesntExisting, inner, errors.NewKV("Collection", colName)) } -// NewErrFailedToStoreIndexedField returns a new error indicating that the indexed field could not be stored. -func NewErrFailedToStoreIndexedField(fieldName string, inner error) error { - return errors.Wrap(errFailedToStoreIndexedField, inner, errors.NewKV("Field", fieldName)) +// NewErrFailedToStoreIndexedField returns a new error indicating that the indexed field +// could not be stored. +func NewErrFailedToStoreIndexedField(key string, inner error) error { + return errors.Wrap(errFailedToStoreIndexedField, inner, errors.NewKV("Key", key)) } -// NewErrFailedToReadStoredIndexDesc returns a new error indicating that the stored index description -// could not be read. +// NewErrFailedToReadStoredIndexDesc returns a new error indicating that the stored index +// description could not be read. func NewErrFailedToReadStoredIndexDesc(inner error) error { return errors.Wrap(errFailedToReadStoredIndexDesc, inner) } @@ -161,12 +164,14 @@ func NewCanNotDeleteIndexedField(inner error) error { return errors.Wrap(errCanNotDeleteIndexedField, inner) } -// NewErrNonZeroIndexIDProvided returns a new error indicating that a non-zero index ID was provided. +// NewErrNonZeroIndexIDProvided returns a new error indicating that a non-zero index ID was +// provided. func NewErrNonZeroIndexIDProvided(indexID uint32) error { return errors.New(errNonZeroIndexIDProvided, errors.NewKV("ID", indexID)) } -// NewErrFailedToGetCollection returns a new error indicating that the collection could not be obtained. +// NewErrFailedToGetCollection returns a new error indicating that the collection could not +// be obtained. func NewErrFailedToGetCollection(name string, inner error) error { return errors.Wrap(errFailedToGetCollection, inner, errors.NewKV("Name", name)) } @@ -353,3 +358,22 @@ func NewErrUnsupportedIndexFieldType(kind client.FieldKind) error { errors.NewKV("Kind", kind), ) } + +// NewErrIndexDescHasNoFields returns a new error indicating that the given index +// description has no fields. +func NewErrIndexDescHasNoFields(desc client.IndexDescription) error { + return errors.New( + errIndexDescriptionHasNoFields, + errors.NewKV("Description", desc), + ) +} + +// NewErrIndexDescHasNonExistingField returns a new error indicating that the given index +// description points to a field that does not exist. +func NewErrIndexDescHasNonExistingField(desc client.IndexDescription, fieldName string) error { + return errors.New( + errIndexDescHasNonExistingField, + errors.NewKV("Description", desc), + errors.NewKV("Field name", fieldName), + ) +} diff --git a/db/index.go b/db/index.go index c664b4a4e2..1180742c5e 100644 --- a/db/index.go +++ b/db/index.go @@ -99,10 +99,16 @@ func NewCollectionIndex( collection client.Collection, desc client.IndexDescription, ) (CollectionIndex, error) { + if len(desc.Fields) == 0 { + return nil, NewErrIndexDescHasNoFields(desc) + } index := &collectionSimpleIndex{collection: collection, desc: desc} schema := collection.Description().Schema - fieldID := schema.GetFieldKey(desc.Fields[0].Name) - field := schema.Fields[fieldID] + fieldID := client.FieldID(schema.GetFieldKey(desc.Fields[0].Name)) + field, foundField := collection.Description().GetFieldByID(fieldID) + if fieldID == client.FieldID(0) || !foundField { + return nil, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name) + } var e error index.convertFunc, e = getFieldValConverter(field.Kind) return index, e @@ -156,8 +162,7 @@ func (i *collectionSimpleIndex) Save( } err = txn.Datastore().Put(ctx, key.ToDS(), []byte{}) if err != nil { - field, _ := i.collection.Description().GetFieldByID(strconv.Itoa(int(key.IndexID))) - return NewErrFailedToStoreIndexedField(field.Name, err) + return NewErrFailedToStoreIndexedField(key.ToDS().String(), err) } return nil } diff --git a/db/index_test.go b/db/index_test.go index c1bd8f6a3c..f84d1cdba6 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -1356,3 +1356,19 @@ func TestDropAllIndexes_ShouldCloseQueryIterator(t *testing.T) { _ = f.users.dropAllIndexes(f.ctx, f.txn) } + +func TestNewCollectionIndex_IfDescriptionHasNoFields_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + desc := getUsersIndexDescOnName() + desc.Fields = nil + _, err := NewCollectionIndex(f.users, desc) + require.ErrorIs(t, err, NewErrIndexDescHasNoFields(desc)) +} + +func TestNewCollectionIndex_IfDescriptionHasNonExistingField_ReturnError(t *testing.T) { + f := newIndexTestFixture(t) + desc := getUsersIndexDescOnName() + desc.Fields[0].Name = "non_existing_field" + _, err := NewCollectionIndex(f.users, desc) + require.ErrorIs(t, err, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name)) +} From 269b67cf73c3d606a3c36220cc0500b6f8cb6cf8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Jun 2023 15:18:46 +0200 Subject: [PATCH 100/120] Separate iteration over keys from deletion --- db/collection_index.go | 19 +++++++++++++---- db/index.go | 45 ++++++++++++++++++++++------------------- db/indexed_docs_test.go | 1 - 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index 2780b80fd7..a6a825e7e8 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -307,10 +307,17 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) error { prefix := core.NewCollectionIndexKey(c.Name(), "") - err := iteratePrefixKeys(ctx, prefix.ToString(), txn.Systemstore(), - func(ctx context.Context, key ds.Key) error { - return txn.Systemstore().Delete(ctx, key) - }) + keys, err := fetchKeysForPrefix(ctx, prefix.ToString(), txn.Systemstore()) + if err != nil { + return err + } + + for _, key := range keys { + err = txn.Systemstore().Delete(ctx, key) + if err != nil { + return err + } + } return err } @@ -489,9 +496,13 @@ func validateIndexDescription(desc client.IndexDescription) error { func generateIndexName(col client.Collection, fields []client.IndexedFieldDescription, inc int) string { sb := strings.Builder{} + // at the moment we support only single field indexes that can be stored only in + // ascending order. This will change once we introduce composite indexes. direction := "ASC" sb.WriteString(col.Name()) sb.WriteByte('_') + // we can safely assume that there is at least one field in the slice + // because we validate it before calling this function sb.WriteString(fields[0].Name) sb.WriteByte('_') sb.WriteString(direction) diff --git a/db/index.go b/db/index.go index 1180742c5e..111ce30b00 100644 --- a/db/index.go +++ b/db/index.go @@ -114,6 +114,8 @@ func NewCollectionIndex( return index, e } +// collectionSimpleIndex is an non-unique index that indexes documents by a single field. +// Single-field indexes store values only in ascending order. type collectionSimpleIndex struct { collection client.Collection desc client.IndexDescription @@ -123,6 +125,8 @@ type collectionSimpleIndex struct { var _ CollectionIndex = (*collectionSimpleIndex)(nil) func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataStoreKey, error) { + // collectionSimpleIndex only supports single field indexes, that's why we + // can safely assume access the first field indexedFieldName := i.desc.Fields[0].Name fieldVal, err := doc.Get(indexedFieldName) isNil := false @@ -184,49 +188,48 @@ func (i *collectionSimpleIndex) Update( return i.Save(ctx, txn, newDoc) } -func iteratePrefixKeys( +func fetchKeysForPrefix( ctx context.Context, prefix string, storage ds.Read, - execFunc func(context.Context, ds.Key) error, -) error { +) ([]ds.Key, error) { q, err := storage.Query(ctx, query.Query{Prefix: prefix}) if err != nil { - return err + return nil, err } + keys := make([]ds.Key, 0) for res := range q.Next() { if res.Error != nil { _ = q.Close() - return res.Error - } - err = execFunc(ctx, ds.NewKey(res.Key)) - if err != nil { - _ = q.Close() - return err + return nil, res.Error } + keys = append(keys, ds.NewKey(res.Key)) } if err = q.Close(); err != nil { - return err + return nil, err } - return nil + return keys, nil } func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn) error { prefixKey := core.IndexDataStoreKey{} prefixKey.CollectionID = i.collection.ID() prefixKey.IndexID = i.desc.ID - err := iteratePrefixKeys(ctx, prefixKey.ToString(), txn.Datastore(), - func(ctx context.Context, key ds.Key) error { - err := txn.Datastore().Delete(ctx, key) - if err != nil { - return NewCanNotDeleteIndexedField(err) - } - return nil - }) + keys, err := fetchKeysForPrefix(ctx, prefixKey.ToString(), txn.Datastore()) + if err != nil { + return err + } + + for _, key := range keys { + err := txn.Datastore().Delete(ctx, key) + if err != nil { + return NewCanNotDeleteIndexedField(err) + } + } - return err + return nil } func (i *collectionSimpleIndex) Name() string { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 63696690b7..063c22dbfc 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -709,7 +709,6 @@ func TestNonUniqueDrop_IfDataStorageFails_ReturnError(t *testing.T) { q.EXPECT().Close().Unset() q.EXPECT().Close().Return(testErr) mockedDS.Query(mock.Anything, mock.Anything).Return(q, nil) - mockedDS.Delete(mock.Anything, mock.Anything).Return(nil) }, }, } From a9b1975f36edbc9b1679877cf8fdf8050604a3f5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Jun 2023 22:48:26 +0200 Subject: [PATCH 101/120] Fix key decomposition --- core/key.go | 4 ---- core/key_test.go | 3 --- 2 files changed, 7 deletions(-) diff --git a/core/key.go b/core/key.go index 50e1400ba9..1e3ef731a9 100644 --- a/core/key.go +++ b/core/key.go @@ -426,10 +426,6 @@ func NewIndexDataStoreKey(key string) (IndexDataStoreKey, error) { indexKey.IndexID = uint32(indID) for i := 2; i < len(elements); i++ { - _, err = strconv.Atoi(elements[i]) - if err != nil { - return IndexDataStoreKey{}, ErrInvalidKey - } indexKey.FieldValues = append(indexKey.FieldValues, []byte(elements[i])) } diff --git a/core/key_test.go b/core/key_test.go index 39b3443fd2..307ad555ee 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -404,12 +404,9 @@ func TestNewIndexDataStoreKey_InvalidKey(t *testing.T) { "/1", "/1/2", " /1/2/3", - "/1/2/3 ", "1/2/3", "/a/2/3", "/1/b/3", - "/1/2/c", - "/1/2/3/d", } for i, key := range keys { _, err := NewIndexDataStoreKey(key) From 1c3e2eeda12ee251a741817ddb8b0940c89a2cae Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Jun 2023 22:55:35 +0200 Subject: [PATCH 102/120] Make order of reading index descriptions deterministic --- db/collection_index.go | 41 ++++++++++++++++++------------ db/index_test.go | 57 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index a6a825e7e8..2fedab79d9 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -73,23 +73,23 @@ func (db *db) getAllCollectionIndexes( ) ([]collectionIndexDescription, error) { prefix := core.NewCollectionIndexKey("", "") - indexMap, err := deserializePrefix[client.IndexDescription](ctx, + deserializedIndexes, err := deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) if err != nil { return nil, err } - indexes := make([]collectionIndexDescription, 0, len(indexMap)) + indexes := make([]collectionIndexDescription, 0, len(deserializedIndexes)) - for indexKeyStr, index := range indexMap { - indexKey, err := core.NewCollectionIndexKeyFromString(indexKeyStr) + for _, indexRec := range deserializedIndexes { + indexKey, err := core.NewCollectionIndexKeyFromString(indexRec.key) if err != nil { return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) } indexes = append(indexes, collectionIndexDescription{ CollectionName: indexKey.CollectionName, - Index: index, + Index: indexRec.element, }) } @@ -102,14 +102,14 @@ func (db *db) getCollectionIndexes( colName string, ) ([]client.IndexDescription, error) { prefix := core.NewCollectionIndexKey(colName, "") - indexMap, err := deserializePrefix[client.IndexDescription](ctx, + deserializedIndexes, err := deserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) if err != nil { return nil, err } - indexes := make([]client.IndexDescription, 0, len(indexMap)) - for _, index := range indexMap { - indexes = append(indexes, index) + indexes := make([]client.IndexDescription, 0, len(deserializedIndexes)) + for _, indexRec := range deserializedIndexes { + indexes = append(indexes, indexRec.element) } return indexes, nil } @@ -328,14 +328,14 @@ func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]Colle } prefix := core.NewCollectionIndexKey(c.Name(), "") - indexDescriptions, err := deserializePrefix[client.IndexDescription]( + deserializedIndexes, err := deserializePrefix[client.IndexDescription]( ctx, prefix.ToString(), txn.Systemstore()) if err != nil { return nil, err } - colIndexes := make([]CollectionIndex, 0, len(indexDescriptions)) - for _, desc := range indexDescriptions { - index, err := NewCollectionIndex(c, desc) + colIndexes := make([]CollectionIndex, 0, len(deserializedIndexes)) + for _, indexRec := range deserializedIndexes { + index, err := NewCollectionIndex(c, indexRec.element) if err != nil { return nil, err } @@ -513,13 +513,22 @@ func generateIndexName(col client.Collection, fields []client.IndexedFieldDescri return sb.String() } -func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Read) (map[string]T, error) { +type deserializedElement[T any] struct { + key string + element T +} + +func deserializePrefix[T any]( + ctx context.Context, + prefix string, + storage ds.Read, +) ([]deserializedElement[T], error) { q, err := storage.Query(ctx, query.Query{Prefix: prefix}) if err != nil { return nil, NewErrFailedToCreateCollectionQuery(err) } - elements := make(map[string]T) + elements := make([]deserializedElement[T], 0) for res := range q.Next() { if res.Error != nil { _ = q.Close() @@ -532,7 +541,7 @@ func deserializePrefix[T any](ctx context.Context, prefix string, storage ds.Rea _ = q.Close() return nil, NewErrInvalidStoredIndex(err) } - elements[res.Key] = element + elements = append(elements, deserializedElement[T]{key: res.Key, element: element}) } if err := q.Close(); err != nil { return nil, err diff --git a/db/index_test.go b/db/index_test.go index f84d1cdba6..ca5bfb7a2f 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -1143,6 +1143,63 @@ func TestCollectionGetIndexes_IfIndexIsDropped_ShouldUpdateCache(t *testing.T) { assert.Len(t, indexes, 0) } +func TestCollectionGetIndexes_ShouldReturnIndexesInOrderedByName(t *testing.T) { + f := newIndexTestFixtureBare(t) + colDesc := client.CollectionDescription{ + Name: "testCollection", + Schema: client.SchemaDescription{ + Fields: []client.FieldDescription{ + { + Name: "_key", + Kind: client.FieldKind_DocKey, + }, + }, + }, + } + const ( + num = 30 + fieldNamePrefix = "field_" + indexNamePrefix = "index_" + ) + + toSuffix := func(i int) string { + return fmt.Sprintf("%02d", i) + } + + for i := 1; i <= num; i++ { + colDesc.Schema.Fields = append(colDesc.Schema.Fields, + client.FieldDescription{ + Name: fieldNamePrefix + toSuffix(i), + Kind: client.FieldKind_STRING, + Typ: client.LWW_REGISTER, + }) + } + + collection := f.createCollection(colDesc) + + for i := 1; i <= num; i++ { + iStr := toSuffix(i) + indexDesc := client.IndexDescription{ + Name: indexNamePrefix + iStr, + Fields: []client.IndexedFieldDescription{ + {Name: fieldNamePrefix + iStr, Direction: client.Ascending}, + }, + } + + _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + require.NoError(t, err) + } + f.commitTxn() + + indexes, err := collection.GetIndexes(f.ctx) + require.NoError(t, err) + require.Len(t, indexes, num) + + for i := 1; i <= num; i++ { + assert.Equal(t, indexNamePrefix+toSuffix(i), indexes[i-1].Name, "i = %d", i) + } +} + func TestDropIndex_ShouldDeleteIndex(t *testing.T) { f := newIndexTestFixture(t) desc := f.createUserCollectionIndexOnName() From b2ecc39ca29695da765bd1c8d771df4180475243 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Jun 2023 22:58:18 +0200 Subject: [PATCH 103/120] Polish --- db/fetcher/encoded_doc.go | 8 ++++---- db/fetcher/fetcher.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/db/fetcher/encoded_doc.go b/db/fetcher/encoded_doc.go index 36a77ccb46..e12fdd1e71 100644 --- a/db/fetcher/encoded_doc.go +++ b/db/fetcher/encoded_doc.go @@ -191,7 +191,7 @@ func convertToInt(propertyName string, untypedValue any) (int64, error) { // @todo: Implement Encoded Document type type encodedDocument struct { key []byte - Properties map[client.FieldDescription]*encProperty + properties map[client.FieldDescription]*encProperty } var _ EncodedDocument = (*encodedDocument)(nil) @@ -202,7 +202,7 @@ func (encdoc *encodedDocument) Key() []byte { // Reset re-initializes the EncodedDocument object. func (encdoc *encodedDocument) Reset(newKey []byte) { - encdoc.Properties = make(map[client.FieldDescription]*encProperty) + encdoc.properties = make(map[client.FieldDescription]*encProperty) encdoc.key = newKey } @@ -213,7 +213,7 @@ func (encdoc *encodedDocument) Decode() (*client.Document, error) { return nil, err } doc := client.NewDocWithKey(key) - for fieldDesc, prop := range encdoc.Properties { + for fieldDesc, prop := range encdoc.properties { ctype, val, err := prop.Decode() if err != nil { return nil, err @@ -232,7 +232,7 @@ func (encdoc *encodedDocument) Decode() (*client.Document, error) { func (encdoc *encodedDocument) DecodeToDoc(mapping *core.DocumentMapping) (core.Doc, error) { doc := mapping.NewDoc() doc.SetKey(string(encdoc.key)) - for fieldDesc, prop := range encdoc.Properties { + for fieldDesc, prop := range encdoc.properties { _, val, err := prop.Decode() if err != nil { return core.Doc{}, err diff --git a/db/fetcher/fetcher.go b/db/fetcher/fetcher.go index b5ac74c330..9b052c24a4 100644 --- a/db/fetcher/fetcher.go +++ b/db/fetcher/fetcher.go @@ -338,7 +338,7 @@ func (df *DocumentFetcher) processKV(kv *core.KeyValue) error { // to better handle dynamic use cases beyond primary indexes. If a // secondary index is provided, we need to extract the indexed/implicit fields // from the KV pair. - df.doc.Properties[fieldDesc] = &encProperty{ + df.doc.properties[fieldDesc] = &encProperty{ Desc: fieldDesc, Raw: kv.Value, } From c0343ad23422cf00f63a2895f0cb48c1c6f1ba26 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 15 Jun 2023 09:35:25 +0200 Subject: [PATCH 104/120] Polish --- db/collection.go | 4 +- db/collection_index.go | 104 ++++++++++++++++++++--------------------- db/errors.go | 12 ++--- db/index.go | 8 ++-- db/index_test.go | 2 +- 5 files changed, 66 insertions(+), 64 deletions(-) diff --git a/db/collection.go b/db/collection.go index a4215cee33..c52e1fa30b 100644 --- a/db/collection.go +++ b/db/collection.go @@ -436,7 +436,7 @@ func (db *db) getCollectionByVersionID( schemaVersionId string, ) (*collection, error) { if schemaVersionId == "" { - return nil, ErrSchemaVersionIdEmpty + return nil, ErrSchemaVersionIDEmpty } key := core.NewCollectionSchemaVersionKey(schemaVersionId) @@ -488,7 +488,7 @@ func (db *db) getCollectionBySchemaID( schemaID string, ) (client.Collection, error) { if schemaID == "" { - return nil, ErrSchemaIdEmpty + return nil, ErrSchemaIDEmpty } key := core.NewCollectionSchemaKey(schemaID) diff --git a/db/collection_index.go b/db/collection_index.go index 2fedab79d9..cc493b5748 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -66,8 +66,8 @@ func (db *db) dropCollectionIndex( return col.DropIndex(ctx, indexName) } -// getAllCollectionIndexes returns all the indexes in the database. -func (db *db) getAllCollectionIndexes( +// getAllIndexes returns all the indexes in the database. +func (db *db) getAllIndexes( ctx context.Context, txn datastore.Txn, ) ([]collectionIndexDescription, error) { @@ -194,6 +194,56 @@ func (c *collection) CreateIndex( return index.Description(), nil } +func (c *collection) createIndex( + ctx context.Context, + txn datastore.Txn, + desc client.IndexDescription, +) (CollectionIndex, error) { + if desc.Name != "" && !schema.IsValidIndexName(desc.Name) { + return nil, schema.NewErrIndexWithInvalidName("!") + } + err := validateIndexDescription(desc) + if err != nil { + return nil, err + } + + err = c.checkExistingFields(ctx, desc.Fields) + if err != nil { + return nil, err + } + + indexKey, err := c.processIndexName(ctx, txn, &desc) + if err != nil { + return nil, err + } + + colSeq, err := c.db.getSequence(ctx, txn, fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, c.ID())) + if err != nil { + return nil, err + } + colID, err := colSeq.next(ctx, txn) + if err != nil { + return nil, err + } + desc.ID = uint32(colID) + + buf, err := json.Marshal(desc) + if err != nil { + return nil, err + } + + err = txn.Systemstore().Put(ctx, indexKey.ToDS(), buf) + if err != nil { + return nil, err + } + colIndex, err := NewCollectionIndex(c, desc) + if err != nil { + return nil, err + } + c.desc.Indexes = append(c.desc.Indexes, colIndex.Description()) + return colIndex, nil +} + func (c *collection) newFetcher() fetcher.Fetcher { if c.fetcherFactory != nil { return c.fetcherFactory() @@ -369,56 +419,6 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, return indexDescriptions, nil } -func (c *collection) createIndex( - ctx context.Context, - txn datastore.Txn, - desc client.IndexDescription, -) (CollectionIndex, error) { - if desc.Name != "" && !schema.IsValidIndexName(desc.Name) { - return nil, schema.NewErrIndexWithInvalidName("!") - } - err := validateIndexDescription(desc) - if err != nil { - return nil, err - } - - err = c.checkExistingFields(ctx, desc.Fields) - if err != nil { - return nil, err - } - - indexKey, err := c.processIndexName(ctx, txn, &desc) - if err != nil { - return nil, err - } - - colSeq, err := c.db.getSequence(ctx, txn, fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, c.ID())) - if err != nil { - return nil, err - } - colID, err := colSeq.next(ctx, txn) - if err != nil { - return nil, err - } - desc.ID = uint32(colID) - - buf, err := json.Marshal(desc) - if err != nil { - return nil, err - } - - err = txn.Systemstore().Put(ctx, indexKey.ToDS(), buf) - if err != nil { - return nil, err - } - colIndex, err := NewCollectionIndex(c, desc) - if err != nil { - return nil, err - } - c.desc.Indexes = append(c.desc.Indexes, colIndex.Description()) - return colIndex, nil -} - func (c *collection) checkExistingFields( ctx context.Context, fields []client.IndexedFieldDescription, diff --git a/db/errors.go b/db/errors.go index 3a92bc79f1..d09197f008 100644 --- a/db/errors.go +++ b/db/errors.go @@ -87,8 +87,8 @@ var ( ErrSchemaFirstFieldDocKey = errors.New("collection schema first field must be a DocKey") ErrCollectionAlreadyExists = errors.New("collection already exists") ErrCollectionNameEmpty = errors.New("collection name can't be empty") - ErrSchemaIdEmpty = errors.New("schema ID can't be empty") - ErrSchemaVersionIdEmpty = errors.New("schema version ID can't be empty") + ErrSchemaIDEmpty = errors.New("schema ID can't be empty") + ErrSchemaVersionIDEmpty = errors.New("schema version ID can't be empty") ErrKeyEmpty = errors.New("key cannot be empty") ErrAddingP2PCollection = errors.New(errAddingP2PCollection) ErrRemovingP2PCollection = errors.New(errRemovingP2PCollection) @@ -147,13 +147,13 @@ func NewErrCanNotReadCollection(colName string, inner error) error { return errors.Wrap(errCollectionDoesntExisting, inner, errors.NewKV("Collection", colName)) } -// NewErrFailedToStoreIndexedField returns a new error indicating that the indexed field +// NewErrFailedToStoreIndexedField returns a new error indicating that the indexed field // could not be stored. func NewErrFailedToStoreIndexedField(key string, inner error) error { return errors.Wrap(errFailedToStoreIndexedField, inner, errors.NewKV("Key", key)) } -// NewErrFailedToReadStoredIndexDesc returns a new error indicating that the stored index +// NewErrFailedToReadStoredIndexDesc returns a new error indicating that the stored index // description could not be read. func NewErrFailedToReadStoredIndexDesc(inner error) error { return errors.Wrap(errFailedToReadStoredIndexDesc, inner) @@ -164,13 +164,13 @@ func NewCanNotDeleteIndexedField(inner error) error { return errors.Wrap(errCanNotDeleteIndexedField, inner) } -// NewErrNonZeroIndexIDProvided returns a new error indicating that a non-zero index ID was +// NewErrNonZeroIndexIDProvided returns a new error indicating that a non-zero index ID was // provided. func NewErrNonZeroIndexIDProvided(indexID uint32) error { return errors.New(errNonZeroIndexIDProvided, errors.NewKV("ID", indexID)) } -// NewErrFailedToGetCollection returns a new error indicating that the collection could not +// NewErrFailedToGetCollection returns a new error indicating that the collection could not // be obtained. func NewErrFailedToGetCollection(name string, inner error) error { return errors.Wrap(errFailedToGetCollection, inner, errors.NewKV("Name", name)) diff --git a/db/index.go b/db/index.go index 111ce30b00..59ff3cb169 100644 --- a/db/index.go +++ b/db/index.go @@ -124,7 +124,9 @@ type collectionSimpleIndex struct { var _ CollectionIndex = (*collectionSimpleIndex)(nil) -func (i *collectionSimpleIndex) getDocKey(doc *client.Document) (core.IndexDataStoreKey, error) { +func (i *collectionSimpleIndex) getDocumentsIndexKey( + doc *client.Document, +) (core.IndexDataStoreKey, error) { // collectionSimpleIndex only supports single field indexes, that's why we // can safely assume access the first field indexedFieldName := i.desc.Fields[0].Name @@ -160,7 +162,7 @@ func (i *collectionSimpleIndex) Save( txn datastore.Txn, doc *client.Document, ) error { - key, err := i.getDocKey(doc) + key, err := i.getDocumentsIndexKey(doc) if err != nil { return err } @@ -177,7 +179,7 @@ func (i *collectionSimpleIndex) Update( oldDoc *client.Document, newDoc *client.Document, ) error { - key, err := i.getDocKey(oldDoc) + key, err := i.getDocumentsIndexKey(oldDoc) if err != nil { return err } diff --git a/db/index_test.go b/db/index_test.go index ca5bfb7a2f..e5420b24b9 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -240,7 +240,7 @@ func (f *indexTestFixture) createCollectionIndexFor( } func (f *indexTestFixture) getAllIndexes() ([]collectionIndexDescription, error) { - return f.db.getAllCollectionIndexes(f.ctx, f.txn) + return f.db.getAllIndexes(f.ctx, f.txn) } func (f *indexTestFixture) getCollectionIndexes(colName string) ([]client.IndexDescription, error) { From 448cf93526a87d99349f94ee9c8ceedcaa37a8a6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 15 Jun 2023 09:43:41 +0200 Subject: [PATCH 105/120] Add missing test --- core/key.go | 1 + core/key_test.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/core/key.go b/core/key.go index 1e3ef731a9..a211387f68 100644 --- a/core/key.go +++ b/core/key.go @@ -233,6 +233,7 @@ func NewCollectionSchemaVersionKey(schemaVersionId string) CollectionSchemaVersi return CollectionSchemaVersionKey{SchemaVersionId: schemaVersionId} } +// NewCollectionIndexKey creates a new CollectionIndexKey from a collection name and index name. func NewCollectionIndexKey(colID, indexName string) CollectionIndexKey { return CollectionIndexKey{CollectionName: colID, IndexName: indexName} } diff --git a/core/key_test.go b/core/key_test.go index 307ad555ee..d22498bd8c 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -310,6 +310,14 @@ func TestIndexDatastoreKey_EqualTrue(t *testing.T) { } } +func TestCollectionIndexKey_Bytes(t *testing.T) { + key := CollectionIndexKey{ + CollectionName: "col", + IndexName: "idx", + } + assert.Equal(t, []byte(COLLECTION_INDEX+"/col/idx"), key.Bytes()) +} + func TestIndexDatastoreKey_EqualFalse(t *testing.T) { cases := [][]IndexDataStoreKey{ { From 6e01caf6f0a1e644d572e7f831f6e08603a2838a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 15 Jun 2023 21:59:51 +0200 Subject: [PATCH 106/120] Store indexed values using CBOR encoding Rely on CBOR's nil-value distinction instead of manual --- db/index.go | 118 ++++++++++++++++++---------------------- db/indexed_docs_test.go | 39 ++++++------- 2 files changed, 69 insertions(+), 88 deletions(-) diff --git a/db/index.go b/db/index.go index 59ff3cb169..702272e759 100644 --- a/db/index.go +++ b/db/index.go @@ -12,7 +12,6 @@ package db import ( "context" - "strconv" "time" ds "github.com/ipfs/go-datastore" @@ -25,11 +24,6 @@ import ( "github.com/sourcenetwork/defradb/errors" ) -const ( - indexFieldValuePrefix = "v" - indexFieldNilValue = "n" -) - // CollectionIndex is an interface for collection indexes // It abstracts away common index functionality to be implemented // by different index types: non-unique, unique, and composite @@ -46,52 +40,41 @@ type CollectionIndex interface { Description() client.IndexDescription } -func getFieldValConverter(kind client.FieldKind) (func(any) ([]byte, error), error) { +func canConvertIndexFieldValue[T any](val any) bool { + _, ok := val.(T) + return ok +} + +func getValidateIndexFieldFunc(kind client.FieldKind) func(any) bool { switch kind { case client.FieldKind_STRING: - return func(val any) ([]byte, error) { - return []byte(val.(string)), nil - }, nil + return canConvertIndexFieldValue[string] case client.FieldKind_INT: - return func(val any) ([]byte, error) { - intVal, ok := val.(int64) - if !ok { - return nil, NewErrInvalidFieldValue(kind, val) - } - return []byte(strconv.FormatInt(intVal, 10)), nil - }, nil + return canConvertIndexFieldValue[int64] case client.FieldKind_FLOAT: - return func(val any) ([]byte, error) { - floatVal, ok := val.(float64) - if !ok { - return nil, NewErrInvalidFieldValue(kind, val) - } - return []byte(strconv.FormatFloat(floatVal, 'f', -1, 64)), nil - }, nil + return canConvertIndexFieldValue[float64] case client.FieldKind_BOOL: - return func(val any) ([]byte, error) { - boolVal, ok := val.(bool) + return canConvertIndexFieldValue[bool] + case client.FieldKind_DATETIME: + return func(val any) bool { + timeStrVal, ok := val.(string) if !ok { - return nil, NewErrInvalidFieldValue(kind, val) - } - var intVal int64 = 0 - if boolVal { - intVal = 1 + return false } - return []byte(strconv.FormatInt(intVal, 10)), nil - }, nil - case client.FieldKind_DATETIME: - return func(val any) ([]byte, error) { - timeStrVal := val.(string) _, err := time.Parse(time.RFC3339, timeStrVal) - if err != nil { - return nil, NewErrInvalidFieldValue(kind, val) - } - return []byte(timeStrVal), nil - }, nil + return err == nil + } default: + return nil + } +} + +func getFieldValidateFunc(kind client.FieldKind) (func(any) bool, error) { + validateFunc := getValidateIndexFieldFunc(kind) + if validateFunc == nil { return nil, NewErrUnsupportedIndexFieldType(kind) } + return validateFunc, nil } // NewCollectionIndex creates a new collection index @@ -110,16 +93,18 @@ func NewCollectionIndex( return nil, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name) } var e error - index.convertFunc, e = getFieldValConverter(field.Kind) + index.fieldDesc = field + index.validateFieldFunc, e = getFieldValidateFunc(field.Kind) return index, e } // collectionSimpleIndex is an non-unique index that indexes documents by a single field. // Single-field indexes store values only in ascending order. type collectionSimpleIndex struct { - collection client.Collection - desc client.IndexDescription - convertFunc func(any) ([]byte, error) + collection client.Collection + desc client.IndexDescription + validateFieldFunc func(any) bool + fieldDesc client.FieldDescription } var _ CollectionIndex = (*collectionSimpleIndex)(nil) @@ -127,36 +112,37 @@ var _ CollectionIndex = (*collectionSimpleIndex)(nil) func (i *collectionSimpleIndex) getDocumentsIndexKey( doc *client.Document, ) (core.IndexDataStoreKey, error) { - // collectionSimpleIndex only supports single field indexes, that's why we - // can safely assume access the first field - indexedFieldName := i.desc.Fields[0].Name - fieldVal, err := doc.Get(indexedFieldName) - isNil := false + fieldValue, err := i.getDocFieldValue(doc) if err != nil { - isNil = errors.Is(err, client.ErrFieldNotExist) - if !isNil { - return core.IndexDataStoreKey{}, nil - } + return core.IndexDataStoreKey{}, err } - var storeValue []byte - if isNil { - storeValue = []byte(indexFieldNilValue) - } else { - data, err := i.convertFunc(fieldVal) - if err != nil { - return core.IndexDataStoreKey{}, err - } - storeValue = []byte(string(indexFieldValuePrefix) + string(data)) - } indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = i.collection.ID() indexDataStoreKey.IndexID = i.desc.ID - indexDataStoreKey.FieldValues = [][]byte{storeValue, - []byte(string(indexFieldValuePrefix) + doc.Key().String())} + indexDataStoreKey.FieldValues = [][]byte{fieldValue, []byte(doc.Key().String())} return indexDataStoreKey, nil } +func (i *collectionSimpleIndex) getDocFieldValue(doc *client.Document) ([]byte, error) { + // collectionSimpleIndex only supports single field indexes, that's why we + // can safely access the first field + indexedFieldName := i.desc.Fields[0].Name + fieldVal, err := doc.GetValue(indexedFieldName) + if err != nil { + if errors.Is(err, client.ErrFieldNotExist) { + return client.NewCBORValue(client.LWW_REGISTER, nil).Bytes() + } else { + return nil, err + } + } + writeableVal, ok := fieldVal.(client.WriteableValue) + if !ok || !i.validateFieldFunc(fieldVal.Value()) { + return nil, NewErrInvalidFieldValue(i.fieldDesc.Kind, writeableVal) + } + return writeableVal.Bytes() +} + func (i *collectionSimpleIndex) Save( ctx context.Context, txn datastore.Txn, diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 063c22dbfc..b9229de52a 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -31,9 +31,6 @@ import ( fetcherMocks "github.com/sourcenetwork/defradb/db/fetcher/mocks" ) -const testValuePrefix = "v" -const testNilValue = "n" - type userDoc struct { Name string `json:"name"` Age int `json:"age"` @@ -160,16 +157,20 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { if b.doc != nil { var fieldBytesVal []byte + var writeableVal client.WriteableValue if len(b.values) == 0 { - fieldVal, err := b.doc.Get(b.fieldName) + fieldVal, err := b.doc.GetValue(b.fieldName) require.NoError(b.f.t, err) - fieldBytesVal = []byte(fmt.Sprintf("%s%v", testValuePrefix, fieldVal)) + var ok bool + writeableVal, ok = fieldVal.(client.WriteableValue) + require.True(b.f.t, ok) } else { - fieldBytesVal = b.values[0] + writeableVal = client.NewCBORValue(client.LWW_REGISTER, b.values[0]) } + fieldBytesVal, err = writeableVal.Bytes() + require.NoError(b.f.t, err) - key.FieldValues = [][]byte{[]byte(fieldBytesVal), - []byte(testValuePrefix + b.doc.Key().String())} + key.FieldValues = [][]byte{fieldBytesVal, []byte(b.doc.Key().String())} } else if len(b.values) > 0 { key.FieldValues = b.values } @@ -288,7 +289,6 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { require.ErrorIs(f.t, err, NewErrFailedToStoreIndexedField("name", nil)) } -// @todo: should store as nil value? func TestNonUnique_IfDocDoesNotHaveIndexedField_SkipIndex(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -418,20 +418,18 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { // FieldVal is the value the index will receive for serialization FieldVal any ShouldFail bool - // Stored is the value that is stored as part of the index value key - Stored string }{ {Name: "invalid int", FieldKind: client.FieldKind_INT, FieldVal: "invalid", ShouldFail: true}, {Name: "invalid float", FieldKind: client.FieldKind_FLOAT, FieldVal: "invalid", ShouldFail: true}, {Name: "invalid bool", FieldKind: client.FieldKind_BOOL, FieldVal: "invalid", ShouldFail: true}, {Name: "invalid datetime", FieldKind: client.FieldKind_DATETIME, FieldVal: nowStr[1:], ShouldFail: true}, - {Name: "valid int", FieldKind: client.FieldKind_INT, FieldVal: 12, Stored: "12"}, - {Name: "valid float", FieldKind: client.FieldKind_FLOAT, FieldVal: 36.654, Stored: "36.654"}, - {Name: "valid bool true", FieldKind: client.FieldKind_BOOL, FieldVal: true, Stored: "1"}, - {Name: "valid bool false", FieldKind: client.FieldKind_BOOL, FieldVal: false, Stored: "0"}, - {Name: "valid datetime string", FieldKind: client.FieldKind_DATETIME, FieldVal: nowStr, Stored: nowStr}, - {Name: "valid empty string", FieldKind: client.FieldKind_STRING, FieldVal: "", Stored: ""}, + {Name: "valid int", FieldKind: client.FieldKind_INT, FieldVal: 12}, + {Name: "valid float", FieldKind: client.FieldKind_FLOAT, FieldVal: 36.654}, + {Name: "valid bool true", FieldKind: client.FieldKind_BOOL, FieldVal: true}, + {Name: "valid bool false", FieldKind: client.FieldKind_BOOL, FieldVal: false}, + {Name: "valid datetime string", FieldKind: client.FieldKind_DATETIME, FieldVal: nowStr}, + {Name: "valid empty string", FieldKind: client.FieldKind_STRING, FieldVal: ""}, } for i, tc := range testCase { @@ -482,9 +480,6 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { require.NoError(f.t, err, assertMsg) keyBuilder := newIndexKeyBuilder(f).Col(collection.Name()).Field("field").Doc(doc) - if tc.Stored != "" { - keyBuilder.Values([]byte(testValuePrefix + tc.Stored)) - } key := keyBuilder.Build() keyStr := key.ToDS() @@ -510,7 +505,7 @@ func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { f.saveDocToCollection(doc, f.users) key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). - Values([]byte(testNilValue)).Build() + Values([]byte(nil)).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -1018,7 +1013,7 @@ func TestNonUpdate_IfIndexedFieldWasNil_ShouldDeleteIt(t *testing.T) { f.saveDocToCollection(doc, f.users) oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). - Values([]byte(testNilValue)).Build() + Values([]byte(nil)).Build() err = doc.Set(usersNameFieldName, "John") require.NoError(f.t, err) From 806602dbaec7281d6352a59f8655ac0bcf5faba1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 16 Jun 2023 17:52:42 +0200 Subject: [PATCH 107/120] Remove index cache from collection Correctly implement Txn --- db/collection.go | 21 +++++----- db/collection_index.go | 86 +++++++++++++++++++---------------------- db/index_test.go | 33 +++------------- db/indexed_docs_test.go | 3 ++ 4 files changed, 57 insertions(+), 86 deletions(-) diff --git a/db/collection.go b/db/collection.go index c52e1fa30b..f553ffea3a 100644 --- a/db/collection.go +++ b/db/collection.go @@ -58,7 +58,6 @@ type collection struct { desc client.CollectionDescription - isIndexCached bool indexes []CollectionIndex fetcherFactory func() fetcher.Fetcher } @@ -451,18 +450,19 @@ func (db *db) getCollectionByVersionID( return nil, err } - indexes, err := db.getCollectionIndexes(ctx, txn, desc.Name) - if err != nil { - return nil, err - } - desc.Indexes = indexes - - return &collection{ + col := &collection{ db: db, desc: desc, colID: desc.ID, schemaID: desc.Schema.SchemaID, - }, nil + } + + err = col.loadIndexes(ctx, txn) + if err != nil { + return nil, err + } + + return col, nil } // getCollectionByName returns an existing collection within the database. @@ -638,7 +638,6 @@ func (c *collection) WithTxn(txn datastore.Txn) client.Collection { desc: c.desc, colID: c.colID, schemaID: c.schemaID, - isIndexCached: c.isIndexCached, indexes: c.indexes, fetcherFactory: c.fetcherFactory, } @@ -823,7 +822,7 @@ func (c *collection) save( isCreate bool, ) (cid.Cid, error) { if !isCreate { - err := c.updateIndex(ctx, txn, doc) + err := c.updateIndexedDoc(ctx, txn, doc) if err != nil { return cid.Undef, err } diff --git a/db/collection_index.go b/db/collection_index.go index cc493b5748..92ec6ffbba 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -96,7 +96,7 @@ func (db *db) getAllIndexes( return indexes, nil } -func (db *db) getCollectionIndexes( +func (db *db) fetchCollectionIndexDescriptions( ctx context.Context, txn datastore.Txn, colName string, @@ -115,11 +115,11 @@ func (db *db) getCollectionIndexes( } func (c *collection) indexNewDoc(ctx context.Context, txn datastore.Txn, doc *client.Document) error { - indexes, err := c.getIndexes(ctx, txn) + err := c.loadIndexes(ctx, txn) if err != nil { return err } - for _, index := range indexes { + for _, index := range c.indexes { err = index.Save(ctx, txn, doc) if err != nil { return err @@ -149,12 +149,12 @@ func (c *collection) collectIndexedFields() []*client.FieldDescription { return fields } -func (c *collection) updateIndex( +func (c *collection) updateIndexedDoc( ctx context.Context, txn datastore.Txn, doc *client.Document, ) error { - _, err := c.getIndexes(ctx, txn) + err := c.loadIndexes(ctx, txn) if err != nil { return err } @@ -179,19 +179,13 @@ func (c *collection) CreateIndex( if err != nil { return client.IndexDescription{}, err } + defer c.discardImplicitTxn(ctx, txn) index, err := c.createIndex(ctx, txn, desc) if err != nil { return client.IndexDescription{}, err } - if c.isIndexCached { - c.indexes = append(c.indexes, index) - } - err = c.indexExistingDocs(ctx, txn, index) - if err != nil { - return client.IndexDescription{}, err - } - return index.Description(), nil + return index.Description(), c.commitImplicitTxn(ctx, txn) } func (c *collection) createIndex( @@ -212,7 +206,7 @@ func (c *collection) createIndex( return nil, err } - indexKey, err := c.processIndexName(ctx, txn, &desc) + indexKey, err := c.generateIndexNameIfNeededAndCreateKey(ctx, txn, &desc) if err != nil { return nil, err } @@ -241,6 +235,11 @@ func (c *collection) createIndex( return nil, err } c.desc.Indexes = append(c.desc.Indexes, colIndex.Description()) + c.indexes = append(c.indexes, colIndex) + err = c.indexExistingDocs(ctx, txn, colIndex) + if err != nil { + return nil, err + } return colIndex, nil } @@ -314,16 +313,25 @@ func (c *collection) indexExistingDocs( } func (c *collection) DropIndex(ctx context.Context, indexName string) error { - key := core.NewCollectionIndexKey(c.Name(), indexName) - txn, err := c.getTxn(ctx, false) if err != nil { return err } - _, err = c.getIndexes(ctx, txn) + defer c.discardImplicitTxn(ctx, txn) + + err = c.dropIndex(ctx, txn, indexName) + if err != nil { + return err + } + return c.commitImplicitTxn(ctx, txn) +} + +func (c *collection) dropIndex(ctx context.Context, txn datastore.Txn, indexName string) error { + err := c.loadIndexes(ctx, txn) if err != nil { return err } + var didFind bool for i := range c.indexes { if c.indexes[i].Name() == indexName { @@ -346,6 +354,7 @@ func (c *collection) DropIndex(ctx context.Context, indexName string) error { break } } + key := core.NewCollectionIndexKey(c.Name(), indexName) err = txn.Systemstore().Delete(ctx, key.ToDS()) if err != nil { return err @@ -372,51 +381,34 @@ func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) erro return err } -func (c *collection) getIndexes(ctx context.Context, txn datastore.Txn) ([]CollectionIndex, error) { - if c.isIndexCached { - return c.indexes, nil - } - - prefix := core.NewCollectionIndexKey(c.Name(), "") - deserializedIndexes, err := deserializePrefix[client.IndexDescription]( - ctx, prefix.ToString(), txn.Systemstore()) +func (c *collection) loadIndexes(ctx context.Context, txn datastore.Txn) error { + indexDescriptions, err := c.db.fetchCollectionIndexDescriptions(ctx, txn, c.Name()) if err != nil { - return nil, err + return err } - colIndexes := make([]CollectionIndex, 0, len(deserializedIndexes)) - for _, indexRec := range deserializedIndexes { - index, err := NewCollectionIndex(c, indexRec.element) + colIndexes := make([]CollectionIndex, 0, len(indexDescriptions)) + for _, indexDesc := range indexDescriptions { + index, err := NewCollectionIndex(c, indexDesc) if err != nil { - return nil, err + return err } colIndexes = append(colIndexes, index) } - - descriptions := make([]client.IndexDescription, 0, len(colIndexes)) - for _, index := range colIndexes { - descriptions = append(descriptions, index.Description()) - } - c.desc.Indexes = descriptions + c.desc.Indexes = indexDescriptions c.indexes = colIndexes - c.isIndexCached = true - return colIndexes, nil + return nil } func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { - txn, err := c.getTxn(ctx, true) + txn, err := c.getTxn(ctx, false) if err != nil { return nil, err } - indexes, err := c.getIndexes(ctx, txn) + err = c.loadIndexes(ctx, txn) if err != nil { return nil, err } - indexDescriptions := make([]client.IndexDescription, 0, len(indexes)) - for _, index := range indexes { - indexDescriptions = append(indexDescriptions, index.Description()) - } - - return indexDescriptions, nil + return c.desc.Indexes, nil } func (c *collection) checkExistingFields( @@ -440,7 +432,7 @@ func (c *collection) checkExistingFields( return nil } -func (c *collection) processIndexName( +func (c *collection) generateIndexNameIfNeededAndCreateKey( ctx context.Context, txn datastore.Txn, desc *client.IndexDescription, diff --git a/db/index_test.go b/db/index_test.go index e5420b24b9..7822a146ab 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -244,7 +244,7 @@ func (f *indexTestFixture) getAllIndexes() ([]collectionIndexDescription, error) } func (f *indexTestFixture) getCollectionIndexes(colName string) ([]client.IndexDescription, error) { - return f.db.getCollectionIndexes(f.ctx, f.txn, colName) + return f.db.fetchCollectionIndexDescriptions(f.ctx, f.txn, colName) } func (f *indexTestFixture) createCollection( @@ -920,23 +920,6 @@ func TestCollectionGetIndexes_ShouldReturnIndexes(t *testing.T) { assert.Equal(t, testUsersColIndexName, indexes[0].Name) } -func TestCollectionGetIndexes_IfCalledAgain_ShouldReturnCached(t *testing.T) { - f := newIndexTestFixture(t) - - f.createUserCollectionIndexOnName() - - _, err := f.users.GetIndexes(f.ctx) - require.NoError(t, err) - - mockedTxn := mocks.NewTxnWithMultistore(f.t) - - indexes, err := f.users.WithTxn(mockedTxn).GetIndexes(f.ctx) - require.NoError(t, err) - - require.Equal(t, 1, len(indexes)) - assert.Equal(t, testUsersColIndexName, indexes[0].Name) -} - func TestCollectionGetIndexes_ShouldCloseQueryIterator(t *testing.T) { f := newIndexTestFixture(t) @@ -957,7 +940,7 @@ func TestCollectionGetIndexes_ShouldCloseQueryIterator(t *testing.T) { assert.NoError(t, err) } -func TestCollectionGetIndexes_IfSystemStoreFails_ShouldNotCache(t *testing.T) { +func TestCollectionGetIndexes_IfSystemStoreFails_ReturnError(t *testing.T) { testErr := errors.New("test error") testCases := []struct { @@ -1010,12 +993,6 @@ func TestCollectionGetIndexes_IfSystemStoreFails_ShouldNotCache(t *testing.T) { _, err := f.users.WithTxn(mockedTxn).GetIndexes(f.ctx) require.ErrorIs(t, err, testCase.ExpectedError) - - indexes, err := f.users.GetIndexes(f.ctx) - require.NoError(t, err) - - require.Equal(t, 1, len(indexes)) - assert.Equal(t, testUsersColIndexName, indexes[0].Name) } } @@ -1100,7 +1077,7 @@ func TestCollectionGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { require.ElementsMatch(t, []uint32{1, 2}, []uint32{indexes[0].ID, indexes[1].ID}) } -func TestCollectionGetIndexes_IfIndexIsCreated_ShouldUpdateCache(t *testing.T) { +func TestCollectionGetIndexes_IfIndexIsCreated_ReturnUpdateIndexes(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -1117,7 +1094,7 @@ func TestCollectionGetIndexes_IfIndexIsCreated_ShouldUpdateCache(t *testing.T) { assert.Len(t, indexes, 2) } -func TestCollectionGetIndexes_IfIndexIsDropped_ShouldUpdateCache(t *testing.T) { +func TestCollectionGetIndexes_IfIndexIsDropped_ReturnUpdateIndexes(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() @@ -1259,7 +1236,7 @@ func TestDropIndex_IfFailsToCreateTxn_ReturnError(t *testing.T) { require.ErrorIs(t, err, testErr) } -func TestDropIndex_IfFailsToDeleteFromStorage_ShouldNotCache(t *testing.T) { +func TestDropIndex_IfFailsToDeleteFromStorage_ReturnError(t *testing.T) { f := newIndexTestFixture(t) f.createUserCollectionIndexOnName() diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index b9229de52a..8ab0cf677f 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -812,6 +812,9 @@ func TestNonUniqueUpdate_IfFailsToReadIndexDescription_ReturnError(t *testing.T) mockedTxn.MockDatastore.EXPECT().Get(mock.Anything, mock.Anything).Unset() mockedTxn.MockDatastore.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte{}, nil) + usersCol.(*collection).fetcherFactory = func() fetcher.Fetcher { + return fetcherMocks.NewStubbedFetcher(t) + } err = usersCol.WithTxn(mockedTxn).Update(f.ctx, doc) require.ErrorIs(t, err, testErr) } From 93cae456abaa2776f0a12a1825a54e6e279f7ff1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 16 Jun 2023 23:23:49 +0200 Subject: [PATCH 108/120] Add integration test for dropping non-existing index --- request/graphql/schema/collection.go | 4 +-- tests/integration/index/create_test.go | 4 --- tests/integration/index/drop_test.go | 49 ++++++++++++++++++++++++-- tests/integration/test_case.go | 4 +++ tests/integration/utils2.go | 7 ++-- 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index fe84abc6b7..dba43e5d74 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -247,9 +247,9 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg } - if dirVal.Value == "ASC" { + if dirVal.Value == string(client.Ascending) { desc.Fields[i].Direction = client.Ascending - } else if dirVal.Value == "DESC" { + } else if dirVal.Value == string(client.Descending) { desc.Fields[i].Direction = client.Descending } } diff --git a/tests/integration/index/create_test.go b/tests/integration/index/create_test.go index 5c58c16bef..77c6cd7d9d 100644 --- a/tests/integration/index/create_test.go +++ b/tests/integration/index/create_test.go @@ -42,14 +42,12 @@ func TestIndexCreateWithCollection_ShouldNotHinderQuerying(t *testing.T) { Request: ` query { Users { - _key Name Age } }`, Results: []map[string]any{ { - "_key": "bae-52b9170d-b77a-5887-b877-cbdbb99b009f", "Name": "John", "Age": uint64(21), }, @@ -91,14 +89,12 @@ func TestIndexCreate_ShouldNotHinderQuerying(t *testing.T) { Request: ` query { Users { - _key Name Age } }`, Results: []map[string]any{ { - "_key": "bae-52b9170d-b77a-5887-b877-cbdbb99b009f", "Name": "John", "Age": uint64(21), }, diff --git a/tests/integration/index/drop_test.go b/tests/integration/index/drop_test.go index 31e2e90b56..aec5418c23 100644 --- a/tests/integration/index/drop_test.go +++ b/tests/integration/index/drop_test.go @@ -45,14 +45,59 @@ func TestIndexDrop_ShouldNotHinderQuerying(t *testing.T) { Request: ` query { Users { - _key Name Age } }`, Results: []map[string]any{ { - "_key": "bae-52b9170d-b77a-5887-b877-cbdbb99b009f", + "Name": "John", + "Age": uint64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} + +func TestIndexDrop_IfIndexDoesNotExist_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + Description: "Creation of index with collection should not hinder querying", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + Name: String + Age: Int + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + // bae-52b9170d-b77a-5887-b877-cbdbb99b009f + Doc: ` + { + "Name": "John", + "Age": 21 + }`, + }, + testUtils.DropIndex{ + CollectionID: 0, + IndexName: "non_existing_index", + ExpectedError: "index with name doesn't exists. Name: non_existing_index", + }, + testUtils.Request{ + Request: ` + query { + Users { + Name + Age + } + }`, + Results: []map[string]any{ + { "Name": "John", "Age": uint64(21), }, diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index a40ba591fc..600c7c59e4 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -197,6 +197,10 @@ type DropIndex struct { // the indexes within the database. IndexID int + // The index name of the secondary index within the collection. + // If it is provided, `IndexID` is ignored. + IndexName string + // Any error expected from the action. Optional. // // String can be a partial, and the test will pass if an error is returned that diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 4621da9aa2..4d96286717 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1064,13 +1064,16 @@ func dropIndex( var expectedErrorRaised bool actionNodes := getNodes(action.NodeID, nodes) for nodeID, collections := range getNodeCollections(action.NodeID, nodeCollections) { - indexDesc := indexes[nodeID][action.CollectionID][action.IndexID] + indexName := action.IndexName + if indexName == "" { + indexName = indexes[nodeID][action.CollectionID][action.IndexID].Name + } err := withRetry( actionNodes, nodeID, func() error { - return collections[action.CollectionID].DropIndex(ctx, indexDesc.Name) + return collections[action.CollectionID].DropIndex(ctx, indexName) }, ) expectedErrorRaised = AssertError(t, testCase.Description, err, action.ExpectedError) From 3b445c32d9983dff785a96c7419b27ca256560a5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 10:14:39 +0200 Subject: [PATCH 109/120] Move directive consts --- request/graphql/schema/collection.go | 20 +++++++------------- request/graphql/schema/types/types.go | 5 +++++ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index dba43e5d74..1c1e446857 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -17,19 +17,13 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" + "github.com/sourcenetwork/defradb/request/graphql/schema/types" "github.com/graphql-go/graphql/language/ast" gqlp "github.com/graphql-go/graphql/language/parser" "github.com/graphql-go/graphql/language/source" ) -const ( - indexDirectiveLabel = "index" - indexDirectivePropName = "name" - indexDirectivePropFields = "fields" - indexDirectivePropDirections = "directions" -) - // FromString parses a GQL SDL string into a set of collection descriptions. func FromString(ctx context.Context, schemaString string) ( []client.CollectionDescription, @@ -110,7 +104,7 @@ func fromAstDefinition( fieldDescriptions = append(fieldDescriptions, tmpFieldsDescriptions...) for _, directive := range field.Directives { - if directive.Name.Value == indexDirectiveLabel { + if directive.Name.Value == types.IndexDirectiveLabel { index, err := fieldIndexFromAST(field, directive) if err != nil { return client.CollectionDescription{}, err @@ -132,7 +126,7 @@ func fromAstDefinition( }) for _, directive := range def.Directives { - if directive.Name.Value == indexDirectiveLabel { + if directive.Name.Value == types.IndexDirectiveLabel { index, err := indexFromAST(directive) if err != nil { return client.CollectionDescription{}, err @@ -181,7 +175,7 @@ func fieldIndexFromAST(field *ast.FieldDefinition, directive *ast.Directive) (cl } for _, arg := range directive.Arguments { switch arg.Name.Value { - case indexDirectivePropName: + case types.IndexDirectivePropName: nameVal, ok := arg.Value.(*ast.StringValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg @@ -202,7 +196,7 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { var directions *ast.ListValue for _, arg := range directive.Arguments { switch arg.Name.Value { - case indexDirectivePropName: + case types.IndexDirectivePropName: nameVal, ok := arg.Value.(*ast.StringValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg @@ -211,7 +205,7 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { if !IsValidIndexName(desc.Name) { return client.IndexDescription{}, ErrIndexWithInvalidArg } - case indexDirectivePropFields: + case types.IndexDirectivePropFields: fieldsVal, ok := arg.Value.(*ast.ListValue) if !ok { return client.IndexDescription{}, ErrIndexWithInvalidArg @@ -225,7 +219,7 @@ func indexFromAST(directive *ast.Directive) (client.IndexDescription, error) { Name: fieldVal.Value, }) } - case indexDirectivePropDirections: + case types.IndexDirectivePropDirections: var ok bool directions, ok = arg.Value.(*ast.ListValue) if !ok { diff --git a/request/graphql/schema/types/types.go b/request/graphql/schema/types/types.go index 530b6c55c5..3db6441a5f 100644 --- a/request/graphql/schema/types/types.go +++ b/request/graphql/schema/types/types.go @@ -23,6 +23,11 @@ const ( ExplainArgSimple string = "simple" ExplainArgExecute string = "execute" ExplainArgDebug string = "debug" + + IndexDirectiveLabel = "index" + IndexDirectivePropName = "name" + IndexDirectivePropFields = "fields" + IndexDirectivePropDirections = "directions" ) var ( From 3af3fd2b1aa7bea3a55401d3c5a5c02a8083edaa Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 11:29:37 +0200 Subject: [PATCH 110/120] Add @index directive to graphql schema --- request/graphql/schema/manager.go | 2 ++ request/graphql/schema/types/types.go | 36 +++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/request/graphql/schema/manager.go b/request/graphql/schema/manager.go index 045a4e33ae..76a5441d70 100644 --- a/request/graphql/schema/manager.go +++ b/request/graphql/schema/manager.go @@ -112,6 +112,8 @@ func defaultMutationType() *gql.Object { func defaultDirectivesType() []*gql.Directive { return []*gql.Directive{ schemaTypes.ExplainDirective, + schemaTypes.IndexDirective, + schemaTypes.IndexFieldDirective, } } diff --git a/request/graphql/schema/types/types.go b/request/graphql/schema/types/types.go index 3db6441a5f..75f91fb2c5 100644 --- a/request/graphql/schema/types/types.go +++ b/request/graphql/schema/types/types.go @@ -52,12 +52,12 @@ var ( Values: gql.EnumValueConfigMap{ ExplainArgSimple: &gql.EnumValueConfig{ Value: ExplainArgSimple, - Description: "Simple explaination - dump of the plan graph.", + Description: "Simple explanation - dump of the plan graph.", }, ExplainArgExecute: &gql.EnumValueConfig{ Value: ExplainArgExecute, - Description: "Deeper explaination - insights gathered by executing the plan graph.", + Description: "Deeper explanation - insights gathered by executing the plan graph.", }, ExplainArgDebug: &gql.EnumValueConfig{ @@ -84,6 +84,38 @@ var ( }, }) + IndexDirective *gql.Directive = gql.NewDirective(gql.DirectiveConfig{ + Name: IndexDirectiveLabel, + Description: "@index is a directive that can be used to create an index on a type.", + Args: gql.FieldConfigArgument{ + IndexDirectivePropName: &gql.ArgumentConfig{ + Type: gql.String, + }, + IndexDirectivePropFields: &gql.ArgumentConfig{ + Type: gql.NewList(gql.String), + }, + IndexDirectivePropDirections: &gql.ArgumentConfig{ + Type: gql.NewList(OrderingEnum), + }, + }, + Locations: []string{ + gql.DirectiveLocationObject, + }, + }) + + IndexFieldDirective *gql.Directive = gql.NewDirective(gql.DirectiveConfig{ + Name: IndexDirectiveLabel, + Description: "@index is a directive that can be used to create an index on a field.", + Args: gql.FieldConfigArgument{ + IndexDirectivePropName: &gql.ArgumentConfig{ + Type: gql.String, + }, + }, + Locations: []string{ + gql.DirectiveLocationField, + }, + }) + // PrimaryDirective @primary is used to indicate the primary // side of a one-to-one relationship. PrimaryDirective = gql.NewDirective(gql.DirectiveConfig{ From 4e639c27041144167f8266ab7eec218ec79d22b7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 11:34:56 +0200 Subject: [PATCH 111/120] Move newFetcher --- db/collection.go | 12 ++++++++++++ db/collection_index.go | 9 --------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/db/collection.go b/db/collection.go index f553ffea3a..2f53ef9bb2 100644 --- a/db/collection.go +++ b/db/collection.go @@ -108,6 +108,18 @@ func (db *db) newCollection(desc client.CollectionDescription) (*collection, err }, nil } +// newFetcher returns a new fetcher instance for this collection. +// If a fetcherFactory is set, it will be used to create the fetcher. +// It's a very simple factory, but it allows us to inject a mock fetcher +// for testing. +func (c *collection) newFetcher() fetcher.Fetcher { + if c.fetcherFactory != nil { + return c.fetcherFactory() + } else { + return new(fetcher.DocumentFetcher) + } +} + // createCollection creates a collection and saves it to the database in its system store. // Note: Collection.ID is an autoincrementing value that is generated by the database. func (db *db) createCollection( diff --git a/db/collection_index.go b/db/collection_index.go index 92ec6ffbba..ee8ba99176 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -24,7 +24,6 @@ import ( "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/db/base" - "github.com/sourcenetwork/defradb/db/fetcher" "github.com/sourcenetwork/defradb/request/graphql/schema" ) @@ -243,14 +242,6 @@ func (c *collection) createIndex( return colIndex, nil } -func (c *collection) newFetcher() fetcher.Fetcher { - if c.fetcherFactory != nil { - return c.fetcherFactory() - } else { - return new(fetcher.DocumentFetcher) - } -} - func (c *collection) iterateAllDocs( ctx context.Context, txn datastore.Txn, From 45608947001703f219feb46ebbf1acf777f1ec57 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 11:54:40 +0200 Subject: [PATCH 112/120] Split tests --- tests/integration/index/create_drop_test.go | 63 +++++++++++++++++++++ tests/integration/index/drop_test.go | 46 --------------- 2 files changed, 63 insertions(+), 46 deletions(-) create mode 100644 tests/integration/index/create_drop_test.go diff --git a/tests/integration/index/create_drop_test.go b/tests/integration/index/create_drop_test.go new file mode 100644 index 0000000000..c8aa2bed55 --- /dev/null +++ b/tests/integration/index/create_drop_test.go @@ -0,0 +1,63 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestIndexDrop_ShouldNotHinderQuerying(t *testing.T) { + test := testUtils.TestCase{ + Description: "Creation of index with collection should not hinder querying", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + Name: String @index + Age: Int + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + // bae-52b9170d-b77a-5887-b877-cbdbb99b009f + Doc: ` + { + "Name": "John", + "Age": 21 + }`, + }, + testUtils.DropIndex{ + CollectionID: 0, + IndexID: 0, + }, + testUtils.Request{ + Request: ` + query { + Users { + Name + Age + } + }`, + Results: []map[string]any{ + { + "Name": "John", + "Age": uint64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} diff --git a/tests/integration/index/drop_test.go b/tests/integration/index/drop_test.go index aec5418c23..6fd78f3691 100644 --- a/tests/integration/index/drop_test.go +++ b/tests/integration/index/drop_test.go @@ -16,52 +16,6 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -func TestIndexDrop_ShouldNotHinderQuerying(t *testing.T) { - test := testUtils.TestCase{ - Description: "Creation of index with collection should not hinder querying", - Actions: []any{ - testUtils.SchemaUpdate{ - Schema: ` - type Users { - Name: String @index - Age: Int - } - `, - }, - testUtils.CreateDoc{ - CollectionID: 0, - // bae-52b9170d-b77a-5887-b877-cbdbb99b009f - Doc: ` - { - "Name": "John", - "Age": 21 - }`, - }, - testUtils.DropIndex{ - CollectionID: 0, - IndexID: 0, - }, - testUtils.Request{ - Request: ` - query { - Users { - Name - Age - } - }`, - Results: []map[string]any{ - { - "Name": "John", - "Age": uint64(21), - }, - }, - }, - }, - } - - testUtils.ExecuteTestCase(t, []string{"Users"}, test) -} - func TestIndexDrop_IfIndexDoesNotExist_ReturnError(t *testing.T) { test := testUtils.TestCase{ Description: "Creation of index with collection should not hinder querying", From 9793ac2daaba032eadb706609933be978a27c130 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 15:13:29 +0200 Subject: [PATCH 113/120] Add integration tests for getting indexes --- tests/integration/index/create_drop_test.go | 2 +- tests/integration/index/create_get_test.go | 61 ++++++++++ tests/integration/index/drop_test.go | 2 +- tests/integration/index/get_test.go | 40 ++++++ .../peer/subscribe/with_add_remove_test.go | 2 +- .../simple/peer/subscribe/with_add_test.go | 6 +- tests/integration/p2p.go | 46 +++---- tests/integration/test_case.go | 21 ++++ tests/integration/utils2.go | 115 +++++++++++++++++- 9 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 tests/integration/index/create_get_test.go create mode 100644 tests/integration/index/get_test.go diff --git a/tests/integration/index/create_drop_test.go b/tests/integration/index/create_drop_test.go index c8aa2bed55..b24eff7cd2 100644 --- a/tests/integration/index/create_drop_test.go +++ b/tests/integration/index/create_drop_test.go @@ -18,7 +18,7 @@ import ( func TestIndexDrop_ShouldNotHinderQuerying(t *testing.T) { test := testUtils.TestCase{ - Description: "Creation of index with collection should not hinder querying", + Description: "Drop index should not hinder querying", Actions: []any{ testUtils.SchemaUpdate{ Schema: ` diff --git a/tests/integration/index/create_get_test.go b/tests/integration/index/create_get_test.go new file mode 100644 index 0000000000..998b80ac22 --- /dev/null +++ b/tests/integration/index/create_get_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + "github.com/sourcenetwork/defradb/client" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestIndexGet_ShouldReturnListOfExistingIndexes(t *testing.T) { + test := testUtils.TestCase{ + Description: "Getting indexes should return list of existing indexes", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users @index(name: "age_index", fields: ["Age"]) { + Name: String @index(name: "name_index") + Age: Int + } + `, + }, + testUtils.GetIndexes{ + CollectionID: 0, + ExpectedIndexes: []client.IndexDescription{ + { + Name: "name_index", + ID: 1, + Fields: []client.IndexedFieldDescription{ + { + Name: "Name", + Direction: client.Ascending, + }, + }, + }, + { + Name: "age_index", + ID: 2, + Fields: []client.IndexedFieldDescription{ + { + Name: "Age", + Direction: client.Ascending, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} diff --git a/tests/integration/index/drop_test.go b/tests/integration/index/drop_test.go index 6fd78f3691..d3a754e440 100644 --- a/tests/integration/index/drop_test.go +++ b/tests/integration/index/drop_test.go @@ -18,7 +18,7 @@ import ( func TestIndexDrop_IfIndexDoesNotExist_ReturnError(t *testing.T) { test := testUtils.TestCase{ - Description: "Creation of index with collection should not hinder querying", + Description: "Drop index should return error if index does not exist", Actions: []any{ testUtils.SchemaUpdate{ Schema: ` diff --git a/tests/integration/index/get_test.go b/tests/integration/index/get_test.go new file mode 100644 index 0000000000..dd28643c34 --- /dev/null +++ b/tests/integration/index/get_test.go @@ -0,0 +1,40 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + "github.com/sourcenetwork/defradb/client" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestIndexGet_IfThereAreNoIndexes_ReturnEmptyList(t *testing.T) { + test := testUtils.TestCase{ + Description: "Getting indexes should return empty list if there are no indexes", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + Name: String + Age: Int + } + `, + }, + testUtils.GetIndexes{ + CollectionID: 0, + ExpectedIndexes: []client.IndexDescription{}, + }, + }, + } + + testUtils.ExecuteTestCase(t, []string{"Users"}, test) +} diff --git a/tests/integration/net/state/simple/peer/subscribe/with_add_remove_test.go b/tests/integration/net/state/simple/peer/subscribe/with_add_remove_test.go index 577ffe7142..6e72e690aa 100644 --- a/tests/integration/net/state/simple/peer/subscribe/with_add_remove_test.go +++ b/tests/integration/net/state/simple/peer/subscribe/with_add_remove_test.go @@ -169,7 +169,7 @@ func TestP2PSubscribeAddSingleAndRemoveErroneous(t *testing.T) { }, testUtils.UnsubscribeToCollection{ NodeID: 1, - CollectionIDs: []int{0, testUtils.NonExistantCollectionID}, + CollectionIDs: []int{0, testUtils.NonExistentCollectionID}, ExpectedError: "datastore: key not found", }, testUtils.CreateDoc{ diff --git a/tests/integration/net/state/simple/peer/subscribe/with_add_test.go b/tests/integration/net/state/simple/peer/subscribe/with_add_test.go index cd8d140191..c1274b0564 100644 --- a/tests/integration/net/state/simple/peer/subscribe/with_add_test.go +++ b/tests/integration/net/state/simple/peer/subscribe/with_add_test.go @@ -198,7 +198,7 @@ func TestP2PSubscribeAddSingleErroneousCollectionID(t *testing.T) { }, testUtils.SubscribeToCollection{ NodeID: 1, - CollectionIDs: []int{testUtils.NonExistantCollectionID}, + CollectionIDs: []int{testUtils.NonExistentCollectionID}, ExpectedError: "datastore: key not found", }, testUtils.CreateDoc{ @@ -242,7 +242,7 @@ func TestP2PSubscribeAddValidAndErroneousCollectionID(t *testing.T) { }, testUtils.SubscribeToCollection{ NodeID: 1, - CollectionIDs: []int{0, testUtils.NonExistantCollectionID}, + CollectionIDs: []int{0, testUtils.NonExistentCollectionID}, ExpectedError: "datastore: key not found", }, testUtils.CreateDoc{ @@ -291,7 +291,7 @@ func TestP2PSubscribeAddValidThenErroneousCollectionID(t *testing.T) { }, testUtils.SubscribeToCollection{ NodeID: 1, - CollectionIDs: []int{testUtils.NonExistantCollectionID}, + CollectionIDs: []int{testUtils.NonExistentCollectionID}, ExpectedError: "datastore: key not found", }, testUtils.CreateDoc{ diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 4d7639d7aa..f310b3059d 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -35,18 +35,18 @@ import ( type ConnectPeers struct { // SourceNodeID is the node ID (index) of the first node to connect. // - // Is completely interchangable with TargetNodeID and which way round + // Is completely interchangeable with TargetNodeID and which way round // these properties are specified is purely cosmetic. SourceNodeID int // TargetNodeID is the node ID (index) of the second node to connect. // - // Is completely interchangable with SourceNodeID and which way round + // Is completely interchangeable with SourceNodeID and which way round // these properties are specified is purely cosmetic. TargetNodeID int } -// ConfigureReplicator confugures a directional replicator relationship between +// ConfigureReplicator configures a directional replicator relationship between // two nodes. // // All document changes made in the source node will be synced to the target node. @@ -61,9 +61,9 @@ type ConfigureReplicator struct { TargetNodeID int } -// NonExistantCollectionID can be used to represent a non-existant collection ID, it will be substituted -// for a non-existant collection ID when used in actions that support this. -const NonExistantCollectionID int = -1 +// NonExistentCollectionID can be used to represent a non-existent collection ID, it will be substituted +// for a non-existent collection ID when used in actions that support this. +const NonExistentCollectionID int = -1 // SubscribeToCollection sets up a subscription on the given node to the given collection. // @@ -78,7 +78,7 @@ type SubscribeToCollection struct { // CollectionIDs are the collection IDs (indexes) of the collections to subscribe to. // - // A [NonExistantCollectionID] may be provided to test non-existant collection IDs. + // A [NonExistentCollectionID] may be provided to test non-existent collection IDs. CollectionIDs []int // Any error expected from the action. Optional. @@ -96,7 +96,7 @@ type UnsubscribeToCollection struct { // CollectionIDs are the collection IDs (indexes) of the collections to unsubscribe from. // - // A [NonExistantCollectionID] may be provided to test non-existant collection IDs. + // A [NonExistentCollectionID] may be provided to test non-existent collection IDs. CollectionIDs []int // Any error expected from the action. Optional. @@ -125,11 +125,11 @@ type WaitForSync struct{} // AnyOf may be used as `Results` field where the value may // be one of several values, yet the value of that field must be the same -// across all nodes due to strong eventual consistancy. +// across all nodes due to strong eventual consistency. type AnyOf []any // connectPeers connects two existing, started, nodes as peers. It returns a channel -// that will recieve an empty struct upon sync completion of all expected peer-sync events. +// that will receive an empty struct upon sync completion of all expected peer-sync events. // // Any errors generated whilst configuring the peers or waiting on sync will result in a test failure. func connectPeers( @@ -155,7 +155,7 @@ func connectPeers( log.Info(ctx, "Bootstrapping with peers", logging.NewKV("Addresses", addrs)) sourceNode.Boostrap(addrs) - // Boostrap triggers a bunch of async stuff for which we have no good way of waiting on. It must be + // Bootstrap triggers a bunch of async stuff for which we have no good way of waiting on. It must be // allowed to complete before documentation begins or it will not even try and sync it. So for now, we // sleep a little. time.Sleep(100 * time.Millisecond) @@ -290,8 +290,8 @@ func collectionSubscribedTo( return false } -// configureReplicator configures a replicator relationship between two existing, staarted, nodes. -// It returns a channel that will recieve an empty struct upon sync completion of all expected +// configureReplicator configures a replicator relationship between two existing, started, nodes. +// It returns a channel that will receive an empty struct upon sync completion of all expected // replicator-sync events. // // Any errors generated whilst configuring the peers or waiting on sync will result in a test failure. @@ -315,10 +315,10 @@ func configureReplicator( _, err = sourceNode.Peer.SetReplicator(ctx, addr) require.NoError(t, err) - return setupRepicatorWaitSync(ctx, t, testCase, 0, cfg, sourceNode, targetNode) + return setupReplicatorWaitSync(ctx, t, testCase, 0, cfg, sourceNode, targetNode) } -func setupRepicatorWaitSync( +func setupReplicatorWaitSync( ctx context.Context, t *testing.T, testCase TestCase, @@ -331,12 +331,12 @@ func setupRepicatorWaitSync( targetToSourceEvents := []int{0} docIDsSyncedToSource := map[int]struct{}{} waitIndex := 0 - currentdocID := 0 + currentDocID := 0 for i := startIndex; i < len(testCase.Actions); i++ { switch action := testCase.Actions[i].(type) { case CreateDoc: if !action.NodeID.HasValue() || action.NodeID.Value() == cfg.SourceNodeID { - docIDsSyncedToSource[currentdocID] = struct{}{} + docIDsSyncedToSource[currentDocID] = struct{}{} } // A document created on the source or one that is created on all nodes will be sent to the target even @@ -345,7 +345,7 @@ func setupRepicatorWaitSync( sourceToTargetEvents[waitIndex] += 1 } - currentdocID++ + currentDocID++ case DeleteDoc: if _, shouldSyncFromTarget := docIDsSyncedToSource[action.DocID]; shouldSyncFromTarget && @@ -411,8 +411,8 @@ func subscribeToCollection( schemaIDs := []string{} for _, collectionIndex := range action.CollectionIDs { - if collectionIndex == NonExistantCollectionID { - schemaIDs = append(schemaIDs, "NonExistantCollectionID") + if collectionIndex == NonExistentCollectionID { + schemaIDs = append(schemaIDs, "NonExistentCollectionID") continue } @@ -445,8 +445,8 @@ func unsubscribeToCollection( schemaIDs := []string{} for _, collectionIndex := range action.CollectionIDs { - if collectionIndex == NonExistantCollectionID { - schemaIDs = append(schemaIDs, "NonExistantCollectionID") + if collectionIndex == NonExistentCollectionID { + schemaIDs = append(schemaIDs, "NonExistentCollectionID") continue } @@ -511,7 +511,7 @@ func waitForSync( // a safety in case the stream hangs - we don't want the tests to run forever. case <-time.After(subscriptionTimeout * 10): - assert.Fail(t, "timeout occured while waiting for data stream", testCase.Description) + assert.Fail(t, "timeout occurred while waiting for data stream", testCase.Description) } } } diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 600c7c59e4..43600613b8 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -208,6 +208,27 @@ type DropIndex struct { ExpectedError string } +// GetIndex will attempt to get the given secondary index from the given collection +// using the collection api. +type GetIndexes struct { + // NodeID may hold the ID (index) of a node to create the secondary index on. + // + // If a value is not provided the indexes will be retrieved from the first nodes. + NodeID immutable.Option[int] + + // The collection for which this indexes should be retrieved. + CollectionID int + + // The expected indexes to be returned. + ExpectedIndexes []client.IndexDescription + + // Any error expected from the action. Optional. + // + // String can be a partial, and the test will pass if an error is returned that + // contains this string. + ExpectedError string +} + // Request represents a standard Defra (GQL) request. type Request struct { // NodeID may hold the ID (index) of a node to execute this request on. diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 4d96286717..9a13b5a087 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -315,7 +315,7 @@ func executeTestCase( // documents are by collection (index), these are not node specific. documents := getDocuments(ctx, t, testCase, collections, startActionIndex) // indexes are by collection (index) - indexes := getIndexes(ctx, collections) + indexes := getAllIndexes(ctx, collections) for i := startActionIndex; i <= endActionIndex; i++ { // declare default database for ease of use @@ -349,7 +349,7 @@ func executeTestCase( // If the db was restarted we need to refresh the collection definitions as the old instances // will reference the old (closed) database instances. collections = getCollections(ctx, t, nodes, collectionNames) - indexes = getIndexes(ctx, collections) + indexes = getAllIndexes(ctx, collections) case ConnectPeers: syncChans = append(syncChans, connectPeers(ctx, t, testCase, action, nodes, nodeAddresses)) @@ -370,13 +370,13 @@ func executeTestCase( updateSchema(ctx, t, nodes, testCase, action) // If the schema was updated we need to refresh the collection definitions. collections = getCollections(ctx, t, nodes, collectionNames) - indexes = getIndexes(ctx, collections) + indexes = getAllIndexes(ctx, collections) case SchemaPatch: patchSchema(ctx, t, nodes, testCase, action) // If the schema was updated we need to refresh the collection definitions. collections = getCollections(ctx, t, nodes, collectionNames) - indexes = getIndexes(ctx, collections) + indexes = getAllIndexes(ctx, collections) case CreateDoc: documents = createDoc(ctx, t, testCase, nodes, collections, documents, action) @@ -393,6 +393,9 @@ func executeTestCase( case DropIndex: dropIndex(ctx, t, testCase, nodes, collections, indexes, action) + case GetIndexes: + getIndexes(ctx, t, testCase, nodes, collections, action) + case TransactionRequest2: txns = executeTransactionRequest(ctx, t, db, txns, testCase, action) @@ -697,7 +700,7 @@ actionLoop: case ConfigureReplicator: // Give the nodes a chance to connect to each other and learn about each other's subscribed topics. time.Sleep(100 * time.Millisecond) - syncChans = append(syncChans, setupRepicatorWaitSync( + syncChans = append(syncChans, setupReplicatorWaitSync( ctx, t, testCase, waitGroupStartIndex, action, nodes[action.SourceNodeID], nodes[action.TargetNodeID], )) } @@ -830,7 +833,7 @@ func getDocuments( return documentsByCollection } -func getIndexes( +func getAllIndexes( ctx context.Context, collections [][]client.Collection, ) [][][]client.IndexDescription { @@ -859,6 +862,106 @@ func getIndexes( return result } +func getIndexes( + ctx context.Context, + t *testing.T, + testCase TestCase, + nodes []*node.Node, + nodeCollections [][]client.Collection, + action GetIndexes, +) { + if len(nodeCollections) == 0 { + return + } + + var expectedErrorRaised bool + actionNodes := getNodes(action.NodeID, nodes) + for nodeID, collections := range getNodeCollections(action.NodeID, nodeCollections) { + err := withRetry( + actionNodes, + nodeID, + func() error { + actualIndexes, err := collections[action.CollectionID].GetIndexes(ctx) + if err != nil { + return err + } + + assertIndexesListsEqual(action.ExpectedIndexes, + actualIndexes, t, testCase.Description) + + return nil + }, + ) + expectedErrorRaised = expectedErrorRaised || + AssertError(t, testCase.Description, err, action.ExpectedError) + } + + assertExpectedErrorRaised(t, testCase.Description, action.ExpectedError, expectedErrorRaised) +} + +func assertIndexesListsEqual( + expectedIndexes []client.IndexDescription, + actualIndexes []client.IndexDescription, + t *testing.T, + testDescription string, +) { + toNames := func(indexes []client.IndexDescription) []string { + names := make([]string, len(indexes)) + for i, index := range indexes { + names[i] = index.Name + } + return names + } + + require.ElementsMatch(t, toNames(expectedIndexes), toNames(actualIndexes), testDescription) + + toMap := func(indexes []client.IndexDescription) map[string]client.IndexDescription { + resultMap := map[string]client.IndexDescription{} + for _, index := range indexes { + resultMap[index.Name] = index + } + return resultMap + } + + expectedMap := toMap(expectedIndexes) + actualMap := toMap(actualIndexes) + for key := range expectedMap { + assertIndexesEqual(expectedMap[key], actualMap[key], t, testDescription) + } +} + +func assertIndexesEqual(expectedIndex, actualIndex client.IndexDescription, + t *testing.T, + testDescription string, +) { + assert.Equal(t, expectedIndex.Name, actualIndex.Name, testDescription) + assert.Equal(t, expectedIndex.ID, actualIndex.ID, testDescription) + + toNames := func(fields []client.IndexedFieldDescription) []string { + names := make([]string, len(fields)) + for i, field := range fields { + names[i] = field.Name + } + return names + } + + require.ElementsMatch(t, toNames(expectedIndex.Fields), toNames(actualIndex.Fields), testDescription) + + toMap := func(fields []client.IndexedFieldDescription) map[string]client.IndexedFieldDescription { + resultMap := map[string]client.IndexedFieldDescription{} + for _, field := range fields { + resultMap[field.Name] = field + } + return resultMap + } + + expectedMap := toMap(expectedIndex.Fields) + actualMap := toMap(actualIndex.Fields) + for key := range expectedMap { + assert.Equal(t, expectedMap[key], actualMap[key], testDescription) + } +} + // updateSchema updates the schema using the given details. func updateSchema( ctx context.Context, From a635cb1fe2434b6607667579d1845d6de69e3160 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 15:54:48 +0200 Subject: [PATCH 114/120] Update copyright --- client/index.go | 2 +- datastore/mocks/utils.go | 2 +- db/collection_index.go | 2 +- db/fetcher/mocks/utils.go | 2 +- db/index.go | 2 +- db/index_test.go | 2 +- db/indexed_docs_test.go | 2 +- request/graphql/schema/index_test.go | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/index.go b/client/index.go index b916875470..47b52f00c5 100644 --- a/client/index.go +++ b/client/index.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go index 55c114ba8d..af91fc6d3a 100644 --- a/datastore/mocks/utils.go +++ b/datastore/mocks/utils.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/db/collection_index.go b/db/collection_index.go index ee8ba99176..e2738ecbe2 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/db/fetcher/mocks/utils.go b/db/fetcher/mocks/utils.go index 8fa96ec7e1..85412e9170 100644 --- a/db/fetcher/mocks/utils.go +++ b/db/fetcher/mocks/utils.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/db/index.go b/db/index.go index 702272e759..28f529be22 100644 --- a/db/index.go +++ b/db/index.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/db/index_test.go b/db/index_test.go index 7822a146ab..587a814847 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 8ab0cf677f..31ae8c1d9f 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index c37f62e8e6..379b84647d 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 Democratized Data Foundation +// Copyright 2023 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. From feb420bf481e546b30f0119ea158467ef3029cf1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 15:58:27 +0200 Subject: [PATCH 115/120] Add a comment --- core/key.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/key.go b/core/key.go index a211387f68..a0a4bf58d6 100644 --- a/core/key.go +++ b/core/key.go @@ -426,6 +426,7 @@ func NewIndexDataStoreKey(key string) (IndexDataStoreKey, error) { } indexKey.IndexID = uint32(indID) + // first 2 elements are the collection and index IDs, the rest are field values for i := 2; i < len(elements); i++ { indexKey.FieldValues = append(indexKey.FieldValues, []byte(elements[i])) } From 6e245024601c6439399424b88cf715bb4e1cd1ed Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 15:59:32 +0200 Subject: [PATCH 116/120] Format --- tests/integration/test_case.go | 2 +- tests/integration/utils2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 43600613b8..0d7ad32cc1 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -218,7 +218,7 @@ type GetIndexes struct { // The collection for which this indexes should be retrieved. CollectionID int - + // The expected indexes to be returned. ExpectedIndexes []client.IndexDescription diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 9a13b5a087..d41d36cdf6 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -944,7 +944,7 @@ func assertIndexesEqual(expectedIndex, actualIndex client.IndexDescription, } return names } - + require.ElementsMatch(t, toNames(expectedIndex.Fields), toNames(actualIndex.Fields), testDescription) toMap := func(fields []client.IndexedFieldDescription) map[string]client.IndexedFieldDescription { From bfcd6b7a9d2b1b84af171a71bbc453b888d6d0b2 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 17:11:14 +0200 Subject: [PATCH 117/120] Add documentation to public methods --- db/collection_index.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/db/collection_index.go b/db/collection_index.go index e2738ecbe2..e033aaa1cd 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -170,6 +170,21 @@ func (c *collection) updateIndexedDoc( return nil } +// CreateIndex creates a new index on the collection. +// +// If the index name is empty, a name will be automatically generated. +// Otherwise its uniqueness will be checked against existing indexes and +// it will be validated with `schema.IsValidIndexName` method. +// +// The provided index description must include at least one field with +// a name that exists in the collection schema. +// Also it's `ID` field must be zero. It will be assigned a unique +// incremental value by the database. +// +// The index description will be stored in the system store. +// +// Once finished, if there are existing documents in the collection, +// the documents will be indexed by the new index. func (c *collection) CreateIndex( ctx context.Context, desc client.IndexDescription, @@ -303,6 +318,11 @@ func (c *collection) indexExistingDocs( }) } +// DropIndex removes an index from the collection. +// +// The index will be removed from the system store. +// +// All index artifacts for existing documents related the index will be removed. func (c *collection) DropIndex(ctx context.Context, indexName string) error { txn, err := c.getTxn(ctx, false) if err != nil { @@ -390,6 +410,7 @@ func (c *collection) loadIndexes(ctx context.Context, txn datastore.Txn) error { return nil } +// GetIndexes returns all indexes for the collection. func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { txn, err := c.getTxn(ctx, false) if err != nil { From 5b4c60d15f8b55dd60bfb23ef8d4710511d8b48c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 17:15:27 +0200 Subject: [PATCH 118/120] Make checking fields case sensitive --- db/collection_index.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/db/collection_index.go b/db/collection_index.go index e033aaa1cd..970c812413 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -173,16 +173,16 @@ func (c *collection) updateIndexedDoc( // CreateIndex creates a new index on the collection. // // If the index name is empty, a name will be automatically generated. -// Otherwise its uniqueness will be checked against existing indexes and +// Otherwise its uniqueness will be checked against existing indexes and // it will be validated with `schema.IsValidIndexName` method. // // The provided index description must include at least one field with // a name that exists in the collection schema. // Also it's `ID` field must be zero. It will be assigned a unique // incremental value by the database. -// +// // The index description will be stored in the system store. -// +// // Once finished, if there are existing documents in the collection, // the documents will be indexed by the new index. func (c *collection) CreateIndex( @@ -430,9 +430,8 @@ func (c *collection) checkExistingFields( collectionFields := c.Description().Schema.Fields for _, field := range fields { found := false - fieldLower := strings.ToLower(field.Name) for _, colField := range collectionFields { - if fieldLower == strings.ToLower(colField.Name) { + if field.Name == colField.Name { found = true break } From b458b6d6ad6cc04d132d408dad8ed37b7441aeb5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 17:27:27 +0200 Subject: [PATCH 119/120] Add more documentation --- db/index.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/db/index.go b/db/index.go index 28f529be22..2c5ea2d6b2 100644 --- a/db/index.go +++ b/db/index.go @@ -143,6 +143,7 @@ func (i *collectionSimpleIndex) getDocFieldValue(doc *client.Document) ([]byte, return writeableVal.Bytes() } +// Save indexes a document by storing the indexed field value. func (i *collectionSimpleIndex) Save( ctx context.Context, txn datastore.Txn, @@ -159,6 +160,8 @@ func (i *collectionSimpleIndex) Save( return nil } +// Update updates indexed field values of an existing document. +// It removes the old document from the index and adds the new one. func (i *collectionSimpleIndex) Update( ctx context.Context, txn datastore.Txn, @@ -200,6 +203,9 @@ func fetchKeysForPrefix( return keys, nil } + +// RemoveAll remove all artifacts of the index from the storage, i.e. all index +// field values for all documents. func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn) error { prefixKey := core.IndexDataStoreKey{} prefixKey.CollectionID = i.collection.ID() @@ -220,10 +226,12 @@ func (i *collectionSimpleIndex) RemoveAll(ctx context.Context, txn datastore.Txn return nil } +// Name returns the name of the index func (i *collectionSimpleIndex) Name() string { return i.desc.Name } +// Description returns the description of the index func (i *collectionSimpleIndex) Description() client.IndexDescription { return i.desc } From 391947221da62cc7be3d0b5515522ac6eb83dc4c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Jun 2023 17:49:22 +0200 Subject: [PATCH 120/120] Add missing test cases --- db/index_test.go | 19 +++++++++++++++++++ db/indexed_docs_test.go | 1 + 2 files changed, 20 insertions(+) diff --git a/db/index_test.go b/db/index_test.go index 587a814847..293519b376 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -1283,6 +1283,25 @@ func TestDropIndex_IfIndexWithNameDoesNotExist_ReturnError(t *testing.T) { require.ErrorIs(t, err, NewErrIndexWithNameDoesNotExists(name)) } +func TestDropIndex_IfSystemStoreFails_ReturnError(t *testing.T) { + testErr := errors.New("test error") + + f := newIndexTestFixture(t) + + f.createUserCollectionIndexOnName() + + mockedTxn := f.mockTxn() + + mockedTxn.MockSystemstore = mocks.NewDSReaderWriter(t) + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Unset() + mockedTxn.MockSystemstore.EXPECT().Query(mock.Anything, mock.Anything).Return(nil, testErr) + mockedTxn.EXPECT().Systemstore().Unset() + mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() + + err := f.users.WithTxn(mockedTxn).DropIndex(f.ctx, testUsersColIndexName) + require.ErrorIs(t, err, testErr) +} + func TestDropAllIndexes_ShouldDeleteAllIndexes(t *testing.T) { f := newIndexTestFixture(t) _, err := f.createCollectionIndexFor(usersColName, client.IndexDescription{ diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 31ae8c1d9f..286633a7a5 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -423,6 +423,7 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { {Name: "invalid float", FieldKind: client.FieldKind_FLOAT, FieldVal: "invalid", ShouldFail: true}, {Name: "invalid bool", FieldKind: client.FieldKind_BOOL, FieldVal: "invalid", ShouldFail: true}, {Name: "invalid datetime", FieldKind: client.FieldKind_DATETIME, FieldVal: nowStr[1:], ShouldFail: true}, + {Name: "invalid datetime type", FieldKind: client.FieldKind_DATETIME, FieldVal: 1, ShouldFail: true}, {Name: "valid int", FieldKind: client.FieldKind_INT, FieldVal: 12}, {Name: "valid float", FieldKind: client.FieldKind_FLOAT, FieldVal: 36.654},