From 14e53ec0dd20e63d6468ff1c624e20569fcbf6a1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 15 Dec 2023 10:56:03 +0100 Subject: [PATCH 01/31] unique index --- db/collection_index.go | 3 ++- db/index_test.go | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/db/collection_index.go b/db/collection_index.go index d4f2e4388d..0557c00609 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -232,7 +232,8 @@ func (c *collection) createIndex( c.indexes = append(c.indexes, colIndex) err = c.indexExistingDocs(ctx, txn, colIndex) if err != nil { - return nil, err + removeErr := colIndex.RemoveAll(ctx, txn) + return nil, errors.Join(err, removeErr) } return colIndex, nil } diff --git a/db/index_test.go b/db/index_test.go index f1323f3812..98b8f62f5d 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -198,6 +198,19 @@ func (f *indexTestFixture) createUserCollectionUniqueIndexOnName() client.IndexD return newDesc } +func makeUnique(indexDesc client.IndexDescription) client.IndexDescription { + indexDesc.Unique = true + return indexDesc +} + +func (f *indexTestFixture) createUserCollectionUniqueIndexOnName() client.IndexDescription { + indexDesc := makeUnique(getUsersIndexDescOnName()) + newDesc, err := f.createCollectionIndexFor(f.users.Name(), indexDesc) + require.NoError(f.t, err) + f.commitTxn() + return newDesc +} + func (f *indexTestFixture) createUserCollectionIndexOnAge() client.IndexDescription { newDesc, err := f.createCollectionIndexFor(f.users.Name().Value(), getUsersIndexDescOnAge()) require.NoError(f.t, err) From f28b785784898138ebbb2c53d9e4da8f114cd1d8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 18 Dec 2023 11:00:17 +0100 Subject: [PATCH 02/31] djust indexKeyBuilder to work with composite indexes --- db/index_test.go | 13 ----- db/indexed_docs_test.go | 110 +++++++++++++++++++++------------------- 2 files changed, 59 insertions(+), 64 deletions(-) diff --git a/db/index_test.go b/db/index_test.go index 98b8f62f5d..f1323f3812 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -198,19 +198,6 @@ func (f *indexTestFixture) createUserCollectionUniqueIndexOnName() client.IndexD return newDesc } -func makeUnique(indexDesc client.IndexDescription) client.IndexDescription { - indexDesc.Unique = true - return indexDesc -} - -func (f *indexTestFixture) createUserCollectionUniqueIndexOnName() client.IndexDescription { - indexDesc := makeUnique(getUsersIndexDescOnName()) - newDesc, err := f.createCollectionIndexFor(f.users.Name(), indexDesc) - require.NoError(f.t, err) - f.commitTxn() - return newDesc -} - func (f *indexTestFixture) createUserCollectionIndexOnAge() client.IndexDescription { newDesc, err := f.createCollectionIndexFor(f.users.Name().Value(), getUsersIndexDescOnAge()) require.NoError(f.t, err) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 8e075b22f8..944ba8a500 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -78,12 +78,12 @@ func (f *indexTestFixture) newProdDoc(id int, price float64, cat string, col cli // The format of the non-unique index key is: "////" // Example: "/5/1/12/bae-61cd6879-63ca-5ca9-8731-470a3c1dac69" type indexKeyBuilder struct { - f *indexTestFixture - colName string - fieldName string - doc *client.Document - values [][]byte - isUnique bool + f *indexTestFixture + colName string + fieldsNames []string + doc *client.Document + values [][]byte + isUnique bool } func newIndexKeyBuilder(f *indexTestFixture) *indexKeyBuilder { @@ -95,11 +95,11 @@ func (b *indexKeyBuilder) Col(colName string) *indexKeyBuilder { return b } -// Field sets the field name for the index key. +// Fields sets the fields' names for the index key. // If the field name is not set, the index key will contain only collection id. // When building a key it will it will find the field id to use in the key. -func (b *indexKeyBuilder) Field(fieldName string) *indexKeyBuilder { - b.fieldName = fieldName +func (b *indexKeyBuilder) Fields(fieldsNames ...string) *indexKeyBuilder { + b.fieldsNames = fieldsNames return b } @@ -145,33 +145,41 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { } key.CollectionID = collection.ID() - if b.fieldName == "" { + if len(b.fieldsNames) == 0 { return key } indexes, err := collection.GetIndexes(b.f.ctx) require.NoError(b.f.t, err) +indexLoop: for _, index := range indexes { - if index.Fields[0].Name == b.fieldName { + if len(index.Fields) == len(b.fieldsNames) { + for i := range index.Fields { + if index.Fields[i].Name != b.fieldsNames[i] { + continue indexLoop + } + } key.IndexID = index.ID - break + break indexLoop } } if b.doc != nil { - var fieldBytesVal []byte - var fieldValue *client.FieldValue - var err error - if len(b.values) == 0 { - fieldValue, err = b.doc.GetValue(b.fieldName) + for i, fieldName := range b.fieldsNames { + var fieldBytesVal []byte + var fieldValue *client.FieldValue + var err error + if len(b.values) == 0 { + fieldValue, err = b.doc.GetValue(fieldName) + require.NoError(b.f.t, err) + } else { + fieldValue = client.NewFieldValue(client.LWW_REGISTER, b.values[i]) + } + fieldBytesVal, err = fieldValue.Bytes() require.NoError(b.f.t, err) - } else { - fieldValue = client.NewFieldValue(client.LWW_REGISTER, b.values[0]) + key.FieldValues = append(key.FieldValues, fieldBytesVal) } - fieldBytesVal, err = fieldValue.Bytes() - require.NoError(b.f.t, err) - key.FieldValues = [][]byte{fieldBytesVal} if !b.isUnique { key.FieldValues = append(key.FieldValues, []byte(b.doc.ID().String())) } @@ -259,7 +267,7 @@ func TestNonUnique_IfDocIsAdded_ShouldBeIndexed(t *testing.T) { doc := f.newUserDoc("John", 21, f.users) f.saveDocToCollection(doc, f.users) - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + key := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -272,7 +280,7 @@ func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { f.createUserCollectionIndexOnName() doc := f.newUserDoc("John", 21, f.users) - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + key := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() mockTxn := f.mockTxn() @@ -349,7 +357,7 @@ func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { doc := f.newUserDoc("John", 21, f.users) f.saveDocToCollection(doc, f.users) - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() + key := newIndexKeyBuilder(f).Col(usersColName).Fields(usersAgeFieldName).Doc(doc).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) require.NoError(t, err) @@ -376,8 +384,8 @@ func TestNonUnique_IfMultipleCollectionsWithIndexes_StoreIndexWithCollectionID(t require.NoError(f.t, err) f.commitTxn() - userDocID := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(userDoc).Build() - prodDocID := newIndexKeyBuilder(f).Col(productsColName).Field(productsCategoryFieldName).Doc(prodDoc).Build() + userDocID := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(userDoc).Build() + prodDocID := newIndexKeyBuilder(f).Col(productsColName).Fields(productsCategoryFieldName).Doc(prodDoc).Build() data, err := f.txn.Datastore().Get(f.ctx, userDocID.ToDS()) require.NoError(t, err) @@ -396,8 +404,8 @@ func TestNonUnique_IfMultipleIndexes_StoreIndexWithIndexID(t *testing.T) { doc := f.newUserDoc("John", 21, f.users) f.saveDocToCollection(doc, f.users) - nameKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() - ageKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() + nameKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() + ageKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersAgeFieldName).Doc(doc).Build() data, err := f.txn.Datastore().Get(f.ctx, nameKey.ToDS()) require.NoError(t, err) @@ -509,7 +517,7 @@ func TestNonUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { f.saveDocToCollection(doc, f.users) - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). + key := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc). Values([]byte(nil)).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) @@ -528,8 +536,8 @@ func TestNonUniqueCreate_ShouldIndexExistingDocs(t *testing.T) { f.createUserCollectionIndexOnName() - key1 := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc1).Build() - key2 := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc2).Build() + key1 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc1).Build() + key2 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc2).Build() data, err := f.txn.Datastore().Get(f.ctx, key1.ToDS()) require.NoError(t, err, key1.ToString()) @@ -600,7 +608,7 @@ func TestNonUniqueCreate_IfUponIndexingExistingDocsFetcherFails_ReturnError(t *t f.saveDocToCollection(doc, f.users) f.users.(*collection).fetcherFactory = tc.PrepareFetcher - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + key := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() _, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) require.ErrorIs(t, err, testError, tc.Name) @@ -655,10 +663,10 @@ func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f.saveDocToCollection(f.newProdDoc(1, 55, "games", products), products) - userNameKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Build() - userAgeKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Build() - userWeightKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersWeightFieldName).Build() - prodCatKey := newIndexKeyBuilder(f).Col(productsColName).Field(productsCategoryFieldName).Build() + userNameKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Build() + userAgeKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersAgeFieldName).Build() + userWeightKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersWeightFieldName).Build() + prodCatKey := newIndexKeyBuilder(f).Col(productsColName).Fields(productsCategoryFieldName).Build() err = f.dropIndex(usersColName, testUsersColIndexAge) require.NoError(f.t, err) @@ -699,7 +707,7 @@ func TestNonUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { f.saveDocToCollection(doc, f.users) for _, tc := range cases { - oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + oldKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() err := doc.Set(usersNameFieldName, tc.NewValue) require.NoError(t, err) @@ -707,7 +715,7 @@ func TestNonUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { require.NoError(t, err) f.commitTxn() - newKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + newKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() _, err = f.txn.Datastore().Get(f.ctx, oldKey.ToDS()) require.Error(t, err) @@ -814,14 +822,14 @@ func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { f.saveDocToCollection(doc, f.users) f.users.(*collection).fetcherFactory = tc.PrepareFetcher - oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + oldKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() err := doc.Set(usersNameFieldName, "Islam") require.NoError(t, err, tc.Name) err = f.users.Update(f.ctx, doc) require.Error(t, err, tc.Name) - newKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + newKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() _, err = f.txn.Datastore().Get(f.ctx, oldKey.ToDS()) require.NoError(t, err, tc.Name) @@ -839,7 +847,7 @@ func TestNonUniqueUpdate_IfFailsToUpdateIndex_ReturnError(t *testing.T) { f.saveDocToCollection(doc, f.users) f.commitTxn() - validKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Doc(doc).Build() + validKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersAgeFieldName).Doc(doc).Build() err := f.txn.Datastore().Delete(f.ctx, validKey.ToDS()) require.NoError(f.t, err) f.commitTxn() @@ -960,7 +968,7 @@ func TestNonUpdate_IfIndexedFieldWasNil_ShouldDeleteIt(t *testing.T) { f.saveDocToCollection(doc, f.users) - oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc). + oldKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc). Values([]byte(nil)).Build() err = doc.Set(usersNameFieldName, "John") @@ -970,7 +978,7 @@ func TestNonUpdate_IfIndexedFieldWasNil_ShouldDeleteIt(t *testing.T) { require.NoError(f.t, err) f.commitTxn() - newKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() + newKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Doc(doc).Build() _, err = f.txn.Datastore().Get(f.ctx, newKey.ToDS()) require.NoError(t, err) @@ -1021,8 +1029,8 @@ func TestUniqueCreate_ShouldIndexExistingDocs(t *testing.T) { f.createUserCollectionUniqueIndexOnName() - key1 := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Unique().Doc(doc1).Build() - key2 := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Unique().Doc(doc2).Build() + key1 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Unique().Doc(doc1).Build() + key2 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Unique().Doc(doc2).Build() data, err := f.txn.Datastore().Get(f.ctx, key1.ToDS()) require.NoError(t, err, key1.ToString()) @@ -1047,7 +1055,7 @@ func TestUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { f.saveDocToCollection(doc, f.users) - key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Unique().Doc(doc). + key := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Unique().Doc(doc). Values([]byte(nil)).Build() data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) @@ -1067,8 +1075,8 @@ func TestUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f.saveDocToCollection(f.newUserDoc("John", 21, users), users) f.saveDocToCollection(f.newUserDoc("Islam", 23, users), users) - userNameKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Build() - userAgeKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersAgeFieldName).Build() + userNameKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Build() + userAgeKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersAgeFieldName).Build() err = f.dropIndex(usersColName, testUsersColIndexAge) require.NoError(f.t, err) @@ -1107,7 +1115,7 @@ func TestUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { f.saveDocToCollection(doc, f.users) for _, tc := range cases { - oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Unique().Doc(doc).Build() + oldKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Unique().Doc(doc).Build() err := doc.Set(usersNameFieldName, tc.NewValue) require.NoError(t, err) @@ -1115,7 +1123,7 @@ func TestUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { require.NoError(t, err) f.commitTxn() - newKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Unique().Doc(doc).Build() + newKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName).Unique().Doc(doc).Build() _, err = f.txn.Datastore().Get(f.ctx, oldKey.ToDS()) require.Error(t, err) From 4fbeebee9d6a89847768dc7ba94e762d654b2a1c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 19 Dec 2023 12:25:01 +0100 Subject: [PATCH 03/31] Remove assertion on DocFetched --- .../query_with_index_combined_filter_test.go | 4 +-- .../query_with_index_only_filter_test.go | 36 +++++++++---------- .../index/query_with_relation_filter_test.go | 22 ++++++------ ...uery_with_unique_index_only_filter_test.go | 30 ++++++++-------- 4 files changed, 46 insertions(+), 46 deletions(-) diff --git a/tests/integration/index/query_with_index_combined_filter_test.go b/tests/integration/index/query_with_index_combined_filter_test.go index 8faf5fa37a..98f8a216d6 100644 --- a/tests/integration/index/query_with_index_combined_filter_test.go +++ b/tests/integration/index/query_with_index_combined_filter_test.go @@ -46,7 +46,7 @@ func TestQueryWithIndex_IfIndexFilterWithRegular_ShouldFilter(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(3).WithFieldFetches(6).WithIndexFetches(3), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(6).WithIndexFetches(3), }, }, } @@ -86,7 +86,7 @@ func TestQueryWithIndex_IfMultipleIndexFiltersWithRegular_ShouldFilter(t *testin }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(6).WithFieldFetches(18), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(18), }, }, } diff --git a/tests/integration/index/query_with_index_only_filter_test.go b/tests/integration/index/query_with_index_only_filter_test.go index 82779c5832..a7bab00aaf 100644 --- a/tests/integration/index/query_with_index_only_filter_test.go +++ b/tests/integration/index/query_with_index_only_filter_test.go @@ -45,7 +45,7 @@ func TestQueryWithIndex_WithNonIndexedFields_ShouldFetchAllOfThem(t *testing.T) }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(2).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(1), }, }, } @@ -79,7 +79,7 @@ func TestQueryWithIndex_WithEqualFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(1).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(1), }, }, } @@ -122,7 +122,7 @@ func TestQueryWithIndex_IfSeveralDocsWithEqFilter_ShouldFetchAll(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(2), }, }, } @@ -157,7 +157,7 @@ func TestQueryWithIndex_WithGreaterThanFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(2).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), }, }, } @@ -193,7 +193,7 @@ func TestQueryWithIndex_WithGreaterOrEqualFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, }, } @@ -228,7 +228,7 @@ func TestQueryWithIndex_WithLessThanFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(2).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), }, }, } @@ -264,7 +264,7 @@ func TestQueryWithIndex_WithLessOrEqualFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, }, } @@ -307,7 +307,7 @@ func TestQueryWithIndex_WithNotEqualFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(9).WithFieldFetches(9).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(9).WithIndexFetches(10), }, }, } @@ -343,7 +343,7 @@ func TestQueryWithIndex_WithInFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(2), }, }, } @@ -386,7 +386,7 @@ func TestQueryWithIndex_IfSeveralDocsWithInFilter_ShouldFetchAll(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(2), }, }, } @@ -424,7 +424,7 @@ func TestQueryWithIndex_WithNotInFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(4).WithFieldFetches(8).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(8).WithIndexFetches(10), }, }, } @@ -485,7 +485,7 @@ func TestQueryWithIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req1), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req2, @@ -496,7 +496,7 @@ func TestQueryWithIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req2), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req3, @@ -507,7 +507,7 @@ func TestQueryWithIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req3), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req4, @@ -517,7 +517,7 @@ func TestQueryWithIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req4), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(2).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), }, testUtils.Request{ Request: req5, @@ -528,7 +528,7 @@ func TestQueryWithIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req5), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req6, @@ -536,7 +536,7 @@ func TestQueryWithIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req6), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(0).WithFieldFetches(0).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(0).WithIndexFetches(10), }, }, } @@ -577,7 +577,7 @@ func TestQueryWithIndex_WithNotLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(7).WithFieldFetches(7).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(7).WithIndexFetches(10), }, }, } diff --git a/tests/integration/index/query_with_relation_filter_test.go b/tests/integration/index/query_with_relation_filter_test.go index 57a43bf69e..d8fb14e6d4 100644 --- a/tests/integration/index/query_with_relation_filter_test.go +++ b/tests/integration/index/query_with_relation_filter_test.go @@ -60,7 +60,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, testUtils.Request{ Request: makeExplainQuery(req1), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(6).WithFieldFetches(9).WithIndexFetches(3), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(9).WithIndexFetches(3), }, testUtils.Request{ Request: req2, @@ -70,7 +70,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, testUtils.Request{ Request: makeExplainQuery(req2), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(3).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(1), }, }, } @@ -122,7 +122,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, testUtils.Request{ Request: makeExplainQuery(req1), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(6).WithFieldFetches(9).WithIndexFetches(3), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(9).WithIndexFetches(3), }, testUtils.Request{ Request: req2, @@ -132,7 +132,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, testUtils.Request{ Request: makeExplainQuery(req2), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(3).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(1), }, }, } @@ -182,7 +182,7 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh }, testUtils.Request{ Request: makeExplainQuery(req1), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(3).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(1), }, testUtils.Request{ Request: req2, @@ -194,7 +194,7 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh }, testUtils.Request{ Request: makeExplainQuery(req2), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(6).WithFieldFetches(9).WithIndexFetches(3), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(9).WithIndexFetches(3), }, }, } @@ -245,7 +245,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio }, testUtils.Request{ Request: makeExplainQuery(req1), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(11).WithFieldFetches(12).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(12).WithIndexFetches(1), }, testUtils.Request{ Request: req2, @@ -257,7 +257,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio }, testUtils.Request{ Request: makeExplainQuery(req2), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(15).WithFieldFetches(18).WithIndexFetches(3), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(18).WithIndexFetches(3), }, }, } @@ -301,7 +301,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedRelationWhileI }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(11).WithFieldFetches(12).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(12).WithIndexFetches(1), }, }, } @@ -368,7 +368,7 @@ func TestQueryWithIndexOnOneToTwoRelation_IfFilterOnIndexedRelation_ShouldFilter }, testUtils.Request{ Request: makeExplainQuery(req1), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(3).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(1), }, testUtils.Request{ Request: req2, @@ -383,7 +383,7 @@ func TestQueryWithIndexOnOneToTwoRelation_IfFilterOnIndexedRelation_ShouldFilter }, testUtils.Request{ Request: makeExplainQuery(req2), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(3).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(1), }, }, } diff --git a/tests/integration/index/query_with_unique_index_only_filter_test.go b/tests/integration/index/query_with_unique_index_only_filter_test.go index ad453409d4..cb9b23ebec 100644 --- a/tests/integration/index/query_with_unique_index_only_filter_test.go +++ b/tests/integration/index/query_with_unique_index_only_filter_test.go @@ -42,7 +42,7 @@ func TestQueryWithUniqueIndex_WithEqualFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(1).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(1), }, }, } @@ -77,7 +77,7 @@ func TestQueryWithUniqueIndex_WithGreaterThanFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(2).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), }, }, } @@ -113,7 +113,7 @@ func TestQueryWithUniqueIndex_WithGreaterOrEqualFilter_ShouldFetch(t *testing.T) }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, }, } @@ -148,7 +148,7 @@ func TestQueryWithUniqueIndex_WithLessThanFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(2).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), }, }, } @@ -184,7 +184,7 @@ func TestQueryWithUniqueIndex_WithLessOrEqualFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, }, } @@ -227,7 +227,7 @@ func TestQueryWithUniqueIndex_WithNotEqualFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(9).WithFieldFetches(9).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(9).WithIndexFetches(10), }, }, } @@ -263,7 +263,7 @@ func TestQueryWithUniqueIndex_WithInFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(2), }, }, } @@ -301,7 +301,7 @@ func TestQueryWithUniqueIndex_WithNotInFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(4).WithFieldFetches(8).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(8).WithIndexFetches(10), }, }, } @@ -362,7 +362,7 @@ func TestQueryWithUniqueIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req1), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req2, @@ -373,7 +373,7 @@ func TestQueryWithUniqueIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req2), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req3, @@ -384,7 +384,7 @@ func TestQueryWithUniqueIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req3), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req4, @@ -394,7 +394,7 @@ func TestQueryWithUniqueIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req4), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(1).WithFieldFetches(2).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), }, testUtils.Request{ Request: req5, @@ -405,7 +405,7 @@ func TestQueryWithUniqueIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req5), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(2).WithFieldFetches(4).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), }, testUtils.Request{ Request: req6, @@ -413,7 +413,7 @@ func TestQueryWithUniqueIndex_WithLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req6), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(0).WithFieldFetches(0).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(0).WithIndexFetches(10), }, }, } @@ -454,7 +454,7 @@ func TestQueryWithUniqueIndex_WithNotLikeFilter_ShouldFetch(t *testing.T) { }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithDocFetches(7).WithFieldFetches(7).WithIndexFetches(10), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(7).WithIndexFetches(10), }, }, } From a44954c7f8697f8fbfbf513948872f97acea6487 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 20 Dec 2023 11:31:18 +0100 Subject: [PATCH 04/31] Make base index iterate all fields --- db/index.go | 74 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/db/index.go b/db/index.go index 59fd25eaa9..9c5419e8b0 100644 --- a/db/index.go +++ b/db/index.go @@ -90,16 +90,20 @@ func NewCollectionIndex( if len(desc.Fields) == 0 { return nil, NewErrIndexDescHasNoFields(desc) } - field, foundField := collection.Schema().GetField(desc.Fields[0].Name) - if !foundField { - return nil, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name) - } base := collectionBaseIndex{collection: collection, desc: desc} - base.fieldDesc = field - var err error - base.validateFieldFunc, err = getFieldValidateFunc(field.Kind) - if err != nil { - return nil, err + base.validateFieldFuncs = make([]func(any) bool, 0, len(desc.Fields)) + base.fieldsDescs = make([]client.FieldDescription, 0, len(desc.Fields)) + for _, fieldDesc := range desc.Fields { + field, foundField := collection.Schema().GetField(fieldDesc.Name) + if !foundField { + return nil, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name) + } + base.fieldsDescs = append(base.fieldsDescs, field) + validateFunc, err := getFieldValidateFunc(field.Kind) + if err != nil { + return nil, err + } + base.validateFieldFuncs = append(base.validateFieldFuncs, validateFunc) } if desc.Unique { return &collectionUniqueIndex{collectionBaseIndex: base}, nil @@ -109,34 +113,44 @@ func NewCollectionIndex( } type collectionBaseIndex struct { - collection client.Collection - desc client.IndexDescription - validateFieldFunc func(any) bool - fieldDesc client.FieldDescription + collection client.Collection + desc client.IndexDescription + validateFieldFuncs []func(any) bool + fieldsDescs []client.FieldDescription } -func (i *collectionBaseIndex) getDocFieldValue(doc *client.Document) ([]byte, error) { - // collectionSimpleIndex only supports single field indexes, that's why we - // can safely access the first field - indexedFieldName := i.desc.Fields[0].Name - fieldVal, err := doc.GetValue(indexedFieldName) - if err != nil { - if errors.Is(err, client.ErrFieldNotExist) { - return client.NewFieldValue(client.LWW_REGISTER, nil).Bytes() - } else { +func (i *collectionBaseIndex) getDocFieldValue(doc *client.Document) ([][]byte, error) { + result := make([][]byte, 0, len(i.fieldsDescs)) + for iter := range i.fieldsDescs { + fieldVal, err := doc.GetValue(i.fieldsDescs[iter].Name) + if err != nil { + if errors.Is(err, client.ErrFieldNotExist) { + valBytes, err := client.NewFieldValue(client.LWW_REGISTER, nil).Bytes() + if err != nil { + return nil, err + } + result = append(result, valBytes) + continue + } else { + return nil, err + } + } + if !i.validateFieldFuncs[iter](fieldVal.Value()) { + return nil, NewErrInvalidFieldValue(i.fieldsDescs[iter].Kind, fieldVal) + } + valBytes, err := fieldVal.Bytes() + if err != nil { return nil, err } + result = append(result, valBytes) } - if !i.validateFieldFunc(fieldVal.Value()) { - return nil, NewErrInvalidFieldValue(i.fieldDesc.Kind, fieldVal) - } - return fieldVal.Bytes() + return result, nil } func (i *collectionBaseIndex) getDocumentsIndexKey( doc *client.Document, ) (core.IndexDataStoreKey, error) { - fieldValue, err := i.getDocFieldValue(doc) + fieldValues, err := i.getDocFieldValue(doc) if err != nil { return core.IndexDataStoreKey{}, err } @@ -144,7 +158,7 @@ func (i *collectionBaseIndex) getDocumentsIndexKey( indexDataStoreKey := core.IndexDataStoreKey{} indexDataStoreKey.CollectionID = i.collection.ID() indexDataStoreKey.IndexID = i.desc.ID - indexDataStoreKey.FieldValues = [][]byte{fieldValue} + indexDataStoreKey.FieldValues = fieldValues return indexDataStoreKey, nil } @@ -289,7 +303,7 @@ func (i *collectionUniqueIndex) Save( func (i *collectionUniqueIndex) newUniqueIndexError( doc *client.Document, ) error { - fieldVal, err := doc.GetValue(i.fieldDesc.Name) + fieldVal, err := doc.GetValue(i.fieldsDescs[0].Name) var val any if err != nil { // If the error is ErrFieldNotExist, we leave `val` as is (e.g. nil) @@ -301,7 +315,7 @@ func (i *collectionUniqueIndex) newUniqueIndexError( val = fieldVal.Value() } - return NewErrCanNotIndexNonUniqueField(doc.ID().String(), i.fieldDesc.Name, val) + return NewErrCanNotIndexNonUniqueField(doc.ID().String(), i.fieldsDescs[0].Name, val) } func (i *collectionUniqueIndex) Update( From 18d6eb5a9ad2ce2843d51dea88d6345e277b5053 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 20 Dec 2023 15:16:22 +0100 Subject: [PATCH 05/31] Implement CRUD for composite indexes --- db/index_test.go | 15 ++ db/indexed_docs_test.go | 135 ++++++++++++++++++ .../index/create_composite_test.go | 76 ++++++++++ tests/integration/test_case.go | 1 + tests/integration/utils2.go | 6 +- 5 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 tests/integration/index/create_composite_test.go diff --git a/db/index_test.go b/db/index_test.go index f1323f3812..16ee222199 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -198,6 +198,21 @@ func (f *indexTestFixture) createUserCollectionUniqueIndexOnName() client.IndexD return newDesc } +func addFieldToIndex(indexDesc client.IndexDescription, fieldName string) client.IndexDescription { + indexDesc.Fields = append(indexDesc.Fields, client.IndexedFieldDescription{ + Name: fieldName, Direction: client.Ascending, + }) + return indexDesc +} + +func (f *indexTestFixture) createUserCollectionIndexOnNameAndAge() client.IndexDescription { + indexDesc := addFieldToIndex(getUsersIndexDescOnName(), usersAgeFieldName) + newDesc, err := f.createCollectionIndexFor(f.users.Name(), indexDesc) + require.NoError(f.t, err) + f.commitTxn() + return newDesc +} + func (f *indexTestFixture) createUserCollectionIndexOnAge() client.IndexDescription { newDesc, err := f.createCollectionIndexFor(f.users.Name().Value(), getUsersIndexDescOnAge()) require.NoError(f.t, err) diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 944ba8a500..17c21c4aae 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -1131,3 +1131,138 @@ func TestUniqueUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { require.NoError(t, err) } } + +func TestCompositeCreate_ShouldIndexExistingDocs(t *testing.T) { + f := newIndexTestFixture(t) + defer f.db.Close() + + doc1 := f.newUserDoc("John", 21) + f.saveDocToCollection(doc1, f.users) + doc2 := f.newUserDoc("Islam", 18) + f.saveDocToCollection(doc2, f.users) + + f.createUserCollectionIndexOnNameAndAge() + + key1 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName, usersAgeFieldName).Doc(doc1).Build() + key2 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName, usersAgeFieldName).Doc(doc2).Build() + + ds := f.txn.Datastore() + data, err := ds.Get(f.ctx, key1.ToDS()) + require.NoError(t, err, key1.ToString()) + assert.Len(t, data, 0) + data, err = f.txn.Datastore().Get(f.ctx, key2.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} + +func TestComposite_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { + f := newIndexTestFixture(t) + defer f.db.Close() + f.createUserCollectionIndexOnNameAndAge() + + docJSON, err := json.Marshal(struct { + Age int `json:"age"` + }{Age: 44}) + require.NoError(f.t, err) + + doc, err := client.NewDocFromJSON(docJSON) + require.NoError(f.t, err) + + f.saveDocToCollection(doc, f.users) + + key := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName, usersAgeFieldName).Doc(doc). + Values([]byte(nil)).Build() + + data, err := f.txn.Datastore().Get(f.ctx, key.ToDS()) + require.NoError(t, err) + assert.Len(t, data, 0) +} + +func TestCompositeDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { + f := newIndexTestFixtureBare(t) + users := f.addUsersCollection() + _, err := f.createCollectionIndexFor(users.Name(), addFieldToIndex(getUsersIndexDescOnName(), usersAgeFieldName)) + require.NoError(f.t, err) + _, err = f.createCollectionIndexFor(users.Name(), addFieldToIndex(getUsersIndexDescOnAge(), usersWeightFieldName)) + require.NoError(f.t, err) + f.commitTxn() + + f.saveDocToCollection(f.newUserDoc("John", 21), users) + f.saveDocToCollection(f.newUserDoc("Islam", 23), users) + + userNameAgeKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName, usersAgeFieldName).Build() + userAgeWeightKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersAgeFieldName, usersWeightFieldName).Build() + + err = f.dropIndex(usersColName, testUsersColIndexAge) + require.NoError(f.t, err) + + assert.Len(t, f.getPrefixFromDataStore(userNameAgeKey.ToString()), 2) + assert.Len(t, f.getPrefixFromDataStore(userAgeWeightKey.ToString()), 0) +} + +func TestCompositeUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { + f := newIndexTestFixture(t) + defer f.db.Close() + f.createUserCollectionIndexOnNameAndAge() + + cases := []struct { + Name string + Field string + NewValue any + Exec func(doc *client.Document) error + }{ + { + Name: "update first", + NewValue: "Islam", + Field: usersNameFieldName, + Exec: func(doc *client.Document) error { + return f.users.Update(f.ctx, doc) + }, + }, + { + Name: "save first", + NewValue: "Andy", + Field: usersNameFieldName, + Exec: func(doc *client.Document) error { + return f.users.Save(f.ctx, doc) + }, + }, + { + Name: "update second", + NewValue: 33, + Field: usersAgeFieldName, + Exec: func(doc *client.Document) error { + return f.users.Update(f.ctx, doc) + }, + }, + { + Name: "save second", + NewValue: 36, + Field: usersAgeFieldName, + Exec: func(doc *client.Document) error { + return f.users.Save(f.ctx, doc) + }, + }, + } + + doc := f.newUserDoc("John", 21) + f.saveDocToCollection(doc, f.users) + + for _, tc := range cases { + oldKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName, usersAgeFieldName).Doc(doc).Build() + + err := doc.Set(tc.Field, tc.NewValue) + require.NoError(t, err) + err = tc.Exec(doc) + require.NoError(t, err) + f.commitTxn() + + newKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName, usersAgeFieldName).Doc(doc).Build() + + _, err = f.txn.Datastore().Get(f.ctx, oldKey.ToDS()) + require.Error(t, err) + _, err = f.txn.Datastore().Get(f.ctx, newKey.ToDS()) + require.NoError(t, err) + f.commitTxn() + } +} diff --git a/tests/integration/index/create_composite_test.go b/tests/integration/index/create_composite_test.go new file mode 100644 index 0000000000..c20b1b1240 --- /dev/null +++ b/tests/integration/index/create_composite_test.go @@ -0,0 +1,76 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + "github.com/sourcenetwork/defradb/client" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestCompositeIndexCreate_WhenCreated_CanRetrieve(t *testing.T) { + test := testUtils.TestCase{ + Description: "create composite index and retrieve it", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "John", + "age": 21 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Shahzad", + "age": 22 + }`, + }, + testUtils.CreateIndex{ + CollectionID: 0, + IndexName: "name_age_index", + FieldsNames: []string{"name", "age"}, + }, + testUtils.GetIndexes{ + CollectionID: 0, + ExpectedIndexes: []client.IndexDescription{ + { + Name: "name_age_index", + ID: 1, + Fields: []client.IndexedFieldDescription{ + { + Name: "name", + Direction: client.Ascending, + }, + { + Name: "age", + Direction: client.Ascending, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 435f1cf9b4..2c0e095fe8 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -246,6 +246,7 @@ type CreateIndex struct { // The names of the fields to index. Used only for composite indexes. FieldsNames []string // The directions of the 'FieldsNames' to index. Used only for composite indexes. + // If not provided all fields will be indexed in ascending order. Directions []client.IndexDirection // If Unique is true, the index will be created as a unique index. diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index b2d3b26893..4031e6342a 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1339,9 +1339,13 @@ func createIndex( } } else if len(action.FieldsNames) > 0 { for i := range action.FieldsNames { + dir := client.Ascending + if len(action.Directions) > i { + dir = action.Directions[i] + } indexDesc.Fields = append(indexDesc.Fields, client.IndexedFieldDescription{ Name: action.FieldsNames[i], - Direction: action.Directions[i], + Direction: dir, }) } } From 0c5794b761fab503ad480a11bdb87730fc6ebe83 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 20 Dec 2023 15:53:05 +0100 Subject: [PATCH 06/31] Fix after rebase --- db/index_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/db/index_test.go b/db/index_test.go index 16ee222199..feb60dbb8e 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -209,7 +209,6 @@ func (f *indexTestFixture) createUserCollectionIndexOnNameAndAge() client.IndexD indexDesc := addFieldToIndex(getUsersIndexDescOnName(), usersAgeFieldName) newDesc, err := f.createCollectionIndexFor(f.users.Name(), indexDesc) require.NoError(f.t, err) - f.commitTxn() return newDesc } From 8d23b1c5162dec8fc3b752384668d72d04c64ec3 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 21 Dec 2023 12:54:04 +0100 Subject: [PATCH 07/31] Make value matcher index-key-unaware --- db/fetcher/indexer_iterators.go | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index aa24605559..1e896e1b3b 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -227,7 +227,7 @@ func (i *inIndexIterator) Close() error { } type errorCheckingFilter struct { - matcher indexMatcher + matcher valueMatcher err error } @@ -240,7 +240,7 @@ func (f *errorCheckingFilter) Filter(e query.Entry) bool { f.err = err return false } - res, err := f.matcher.Match(indexKey) + res, err := f.matcher.Match(indexKey.FieldValues[0]) if err != nil { f.err = err return false @@ -251,19 +251,19 @@ func (f *errorCheckingFilter) Filter(e query.Entry) bool { // execInfoIndexMatcherDecorator is a decorator for indexMatcher that counts the number // of indexes fetched on every call to Match. type execInfoIndexMatcherDecorator struct { - matcher indexMatcher + matcher valueMatcher execInfo *ExecInfo } -func (d *execInfoIndexMatcherDecorator) Match(key core.IndexDataStoreKey) (bool, error) { +func (d *execInfoIndexMatcherDecorator) Match(value []byte) (bool, error) { d.execInfo.IndexesFetched++ - return d.matcher.Match(key) + return d.matcher.Match(value) } type scanningIndexIterator struct { queryResultIterator indexKey core.IndexDataStoreKey - matcher indexMatcher + matcher valueMatcher filter errorCheckingFilter execInfo *ExecInfo } @@ -291,9 +291,9 @@ func (i *scanningIndexIterator) Next() (indexIterResult, error) { return res, err } -// checks if the stored index value satisfies the condition -type indexMatcher interface { - Match(core.IndexDataStoreKey) (bool, error) +// checks if the value satisfies the condition +type valueMatcher interface { + Match([]byte) (bool, error) } // indexByteValuesMatcher is a filter that compares the index value with a given value. @@ -304,8 +304,8 @@ type indexByteValuesMatcher struct { evalFunc func(int) bool } -func (m *indexByteValuesMatcher) Match(key core.IndexDataStoreKey) (bool, error) { - res := bytes.Compare(key.FieldValues[0], m.value) +func (m *indexByteValuesMatcher) Match(value []byte) (bool, error) { + res := bytes.Compare(value, m.value) return m.evalFunc(res), nil } @@ -314,8 +314,8 @@ type neIndexMatcher struct { value []byte } -func (m *neIndexMatcher) Match(key core.IndexDataStoreKey) (bool, error) { - return !bytes.Equal(key.FieldValues[0], m.value), nil +func (m *neIndexMatcher) Match(value []byte) (bool, error) { + return !bytes.Equal(value, m.value), nil } // checks if the index value is or is not in the given array @@ -332,8 +332,8 @@ func newNinIndexCmp(values [][]byte, isIn bool) *indexInArrayMatcher { return &indexInArrayMatcher{values: valuesMap, isIn: isIn} } -func (m *indexInArrayMatcher) Match(key core.IndexDataStoreKey) (bool, error) { - _, found := m.values[string(key.FieldValues[0])] +func (m *indexInArrayMatcher) Match(value []byte) (bool, error) { + _, found := m.values[string(value)] return found == m.isIn, nil } @@ -368,9 +368,9 @@ func newLikeIndexCmp(filterValue string, isLike bool) *indexLikeMatcher { return matcher } -func (m *indexLikeMatcher) Match(key core.IndexDataStoreKey) (bool, error) { +func (m *indexLikeMatcher) Match(value []byte) (bool, error) { var currentVal string - err := cbor.Unmarshal(key.FieldValues[0], ¤tVal) + err := cbor.Unmarshal(value, ¤tVal) if err != nil { return false, err } From e06e231c39738b717f31196abb74119911d0c27e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 22 Dec 2023 12:37:02 +0100 Subject: [PATCH 08/31] Make SplitFilter work with multiple fields --- planner/filter/split.go | 20 ++++++++++++++---- planner/filter/split_test.go | 40 ++++++++++++++++++++++++++++++------ planner/scan.go | 2 +- planner/type_join.go | 2 +- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/planner/filter/split.go b/planner/filter/split.go index 1ef153746b..f7ff705b68 100644 --- a/planner/filter/split.go +++ b/planner/filter/split.go @@ -13,7 +13,9 @@ import ( "github.com/sourcenetwork/defradb/planner/mapper" ) -// SplitByField splits the provided filter into 2 filters based on field. +// SplitByFields splits the provided filter into 2 filters based on fields. +// It extract the conditions that apply to the provided fields and returns them +// as the second returned filter. // It can be used for extracting a supType // Eg. (filter: {age: 10, name: "bob", author: {birthday: "June 26, 1990", ...}, ...}) // @@ -22,13 +24,23 @@ import ( // // And the subType filter is the conditions that apply to the queried sub type // ie: {birthday: "June 26, 1990", ...}. -func SplitByField(filter *mapper.Filter, field mapper.Field) (*mapper.Filter, *mapper.Filter) { +func SplitByFields(filter *mapper.Filter, fields ...mapper.Field) (*mapper.Filter, *mapper.Filter) { if filter == nil { return nil, nil } - splitF := CopyField(filter, field) - RemoveField(filter, field) + if len(fields) == 0 { + return filter, nil + } + + splitF := CopyField(filter, fields[0]) + RemoveField(filter, fields[0]) + + for _, field := range fields[1:] { + newSplitF := CopyField(filter, field) + splitF.Conditions = Merge(splitF.Conditions, newSplitF.Conditions) + RemoveField(filter, field) + } if len(filter.Conditions) == 0 { filter = nil diff --git a/planner/filter/split_test.go b/planner/filter/split_test.go index 86fbb0b44a..9ed3700f22 100644 --- a/planner/filter/split_test.go +++ b/planner/filter/split_test.go @@ -21,7 +21,7 @@ import ( func TestSplitFilter(t *testing.T) { tests := []struct { name string - inputField mapper.Field + inputFields []mapper.Field inputFilter map[string]any expectedFilter1 map[string]any expectedFilter2 map[string]any @@ -32,7 +32,7 @@ func TestSplitFilter(t *testing.T) { "name": m("_eq", "John"), "age": m("_gt", 55), }, - inputField: mapper.Field{Index: authorAgeInd}, + inputFields: []mapper.Field{{Index: authorAgeInd}}, expectedFilter1: m("name", m("_eq", "John")), expectedFilter2: m("age", m("_gt", 55)), }, @@ -41,7 +41,7 @@ func TestSplitFilter(t *testing.T) { inputFilter: map[string]any{ "age": m("_gt", 55), }, - inputField: mapper.Field{Index: authorAgeInd}, + inputFields: []mapper.Field{{Index: authorAgeInd}}, expectedFilter1: nil, expectedFilter2: m("age", m("_gt", 55)), }, @@ -50,17 +50,33 @@ func TestSplitFilter(t *testing.T) { inputFilter: map[string]any{ "name": m("_eq", "John"), }, - inputField: mapper.Field{Index: authorAgeInd}, + inputFields: []mapper.Field{{Index: authorAgeInd}}, expectedFilter1: m("name", m("_eq", "John")), expectedFilter2: nil, }, + { + name: "split by 2 fields", + inputFilter: map[string]any{ + "name": m("_eq", "John"), + "age": m("_gt", 55), + "published": m("_eq", true), + "verified": m("_eq", false), + }, + inputFields: []mapper.Field{{Index: authorNameInd}, {Index: authorAgeInd}, {Index: authorVerifiedInd}}, + expectedFilter1: m("published", m("_eq", true)), + expectedFilter2: map[string]any{ + "name": m("_eq", "John"), + "age": m("_gt", 55), + "verified": m("_eq", false), + }, + }, } mapping := getDocMapping() for _, test := range tests { t.Run(test.name, func(t *testing.T) { inputFilter := mapper.ToFilter(request.Filter{Conditions: test.inputFilter}, mapping) - actualFilter1, actualFilter2 := SplitByField(inputFilter, test.inputField) + actualFilter1, actualFilter2 := SplitByFields(inputFilter, test.inputFields...) expectedFilter1 := mapper.ToFilter(request.Filter{Conditions: test.expectedFilter1}, mapping) expectedFilter2 := mapper.ToFilter(request.Filter{Conditions: test.expectedFilter2}, mapping) if expectedFilter1 != nil || actualFilter1 != nil { @@ -73,8 +89,20 @@ func TestSplitFilter(t *testing.T) { } } +func TestSplitFilter_WithNoFields_ReturnsInputFilter(t *testing.T) { + mapping := getDocMapping() + inputFilterConditions := map[string]any{ + "name": m("_eq", "John"), + "age": m("_gt", 55), + } + inputFilter := mapper.ToFilter(request.Filter{Conditions: inputFilterConditions}, mapping) + actualFilter1, actualFilter2 := SplitByFields(inputFilter) + AssertEqualFilterMap(t, inputFilter.Conditions, actualFilter1.Conditions) + assert.Nil(t, actualFilter2) +} + func TestSplitNullFilter(t *testing.T) { - actualFilter1, actualFilter2 := SplitByField(nil, mapper.Field{Index: authorAgeInd}) + actualFilter1, actualFilter2 := SplitByFields(nil, mapper.Field{Index: authorAgeInd}) assert.Nil(t, actualFilter1) assert.Nil(t, actualFilter2) } diff --git a/planner/scan.go b/planner/scan.go index f9edb77b54..3a44a812c9 100644 --- a/planner/scan.go +++ b/planner/scan.go @@ -150,7 +150,7 @@ func (scan *scanNode) initFetcher( typeIndex := scan.documentMapping.FirstIndexOfName(indexedField.Value().Name) field := mapper.Field{Index: typeIndex, Name: indexedField.Value().Name} var indexFilter *mapper.Filter - scan.filter, indexFilter = filter.SplitByField(scan.filter, field) + scan.filter, indexFilter = filter.SplitByFields(scan.filter, field) if indexFilter != nil { fieldDesc, _ := scan.col.Schema().GetField(indexedField.Value().Name) f = fetcher.NewIndexFetcher(f, fieldDesc, indexFilter) diff --git a/planner/type_join.go b/planner/type_join.go index 28d044938f..0d79b76ccd 100644 --- a/planner/type_join.go +++ b/planner/type_join.go @@ -359,7 +359,7 @@ func prepareScanNodeFilterForTypeJoin( filter.RemoveField(scan.filter, subType.Field) } else { var parentFilter *mapper.Filter - scan.filter, parentFilter = filter.SplitByField(scan.filter, subType.Field) + scan.filter, parentFilter = filter.SplitByFields(scan.filter, subType.Field) if parentFilter != nil { if parent.filter == nil { parent.filter = parentFilter From c3c6896522347735dad3548ff220f0acc98b61eb Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 Jan 2024 08:30:16 +0100 Subject: [PATCH 09/31] Swap expected and actual --- tests/integration/explain_result_asserter.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/explain_result_asserter.go b/tests/integration/explain_result_asserter.go index 30126d4fe4..45f998e481 100644 --- a/tests/integration/explain_result_asserter.go +++ b/tests/integration/explain_result_asserter.go @@ -59,15 +59,15 @@ func (a *ExplainResultAsserter) Assert(t *testing.T, result []dataMap) { require.Len(t, result, 1, "Expected len(result) = 1, got %d", len(result)) explainNode, ok := result[0]["explain"].(dataMap) require.True(t, ok, "Expected explain none") - assert.Equal(t, explainNode["executionSuccess"], true, "Expected executionSuccess property") + assert.Equal(t, true, explainNode["executionSuccess"], "Expected executionSuccess property") if a.sizeOfResults.HasValue() { actual := explainNode["sizeOfResult"] - assert.Equal(t, actual, a.sizeOfResults.Value(), + assert.Equal(t, a.sizeOfResults.Value(), actual, "Expected %d sizeOfResult, got %d", a.sizeOfResults.Value(), actual) } if a.planExecutions.HasValue() { actual := explainNode["planExecutions"] - assert.Equal(t, actual, a.planExecutions.Value(), + assert.Equal(t, a.planExecutions.Value(), actual, "Expected %d planExecutions, got %d", a.planExecutions.Value(), actual) } selectTopNode, ok := explainNode["selectTopNode"].(dataMap) @@ -78,7 +78,7 @@ func (a *ExplainResultAsserter) Assert(t *testing.T, result []dataMap) { if a.filterMatches.HasValue() { filterMatches, hasFilterMatches := selectNode["filterMatches"] require.True(t, hasFilterMatches, "Expected filterMatches property") - assert.Equal(t, filterMatches, uint64(a.filterMatches.Value()), + assert.Equal(t, uint64(a.filterMatches.Value()), filterMatches, "Expected %d filterMatches, got %d", a.filterMatches, filterMatches) } @@ -102,22 +102,22 @@ func (a *ExplainResultAsserter) Assert(t *testing.T, result []dataMap) { if a.iterations.HasValue() { actual := getScanNodesProp(iterationsProp) - assert.Equal(t, actual, uint64(a.iterations.Value()), + assert.Equal(t, uint64(a.iterations.Value()), actual, "Expected %d iterations, got %d", a.iterations.Value(), actual) } if a.docFetches.HasValue() { actual := getScanNodesProp(docFetchesProp) - assert.Equal(t, actual, uint64(a.docFetches.Value()), + assert.Equal(t, uint64(a.docFetches.Value()), actual, "Expected %d docFetches, got %d", a.docFetches.Value(), actual) } if a.fieldFetches.HasValue() { actual := getScanNodesProp(fieldFetchesProp) - assert.Equal(t, actual, uint64(a.fieldFetches.Value()), + assert.Equal(t, uint64(a.fieldFetches.Value()), actual, "Expected %d fieldFetches, got %d", a.fieldFetches.Value(), actual) } if a.indexFetches.HasValue() { actual := getScanNodesProp(indexFetchesProp) - assert.Equal(t, actual, uint64(a.indexFetches.Value()), + assert.Equal(t, uint64(a.indexFetches.Value()), actual, "Expected %d indexFetches, got %d", a.indexFetches.Value(), actual) } } From 640e2b926596246b34f60426892f8628b3b15074 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 Jan 2024 11:14:14 +0100 Subject: [PATCH 10/31] Pass around index instead of indexed field --- client/index.go | 10 ++ client/index_test.go | 119 ++++++++++++++++++ planner/planner.go | 11 +- planner/scan.go | 11 +- planner/select.go | 22 ++-- planner/type_join.go | 4 +- .../query_with_index_combined_filter_test.go | 37 ++++++ 7 files changed, 191 insertions(+), 23 deletions(-) create mode 100644 client/index_test.go diff --git a/client/index.go b/client/index.go index 5e2d397394..7fb59658b5 100644 --- a/client/index.go +++ b/client/index.go @@ -58,3 +58,13 @@ func (d CollectionDescription) CollectIndexedFields(schema *SchemaDescription) [ } return fields } + +func (d CollectionDescription) CollectIndexesOnField(fieldName string) []IndexDescription { + result := []IndexDescription{} + for _, index := range d.Indexes { + if index.Fields[0].Name == fieldName { + result = append(result, index) + } + } + return result +} diff --git a/client/index_test.go b/client/index_test.go new file mode 100644 index 0000000000..cc5c6d79a7 --- /dev/null +++ b/client/index_test.go @@ -0,0 +1,119 @@ +package client + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCollectIndexesOnField(t *testing.T) { + tests := []struct { + name string + desc CollectionDescription + field string + expected []IndexDescription + }{ + { + name: "no indexes", + desc: CollectionDescription{ + Indexes: []IndexDescription{}, + }, + field: "test", + expected: []IndexDescription{}, + }, + { + name: "single index on field", + desc: CollectionDescription{ + Indexes: []IndexDescription{ + { + Name: "index1", + Fields: []IndexedFieldDescription{ + {Name: "test", Direction: Ascending}, + }, + }, + }, + }, + field: "test", + expected: []IndexDescription{ + { + Name: "index1", + Fields: []IndexedFieldDescription{ + {Name: "test", Direction: Ascending}, + }, + }, + }, + }, + { + name: "multiple indexes on field", + desc: CollectionDescription{ + Indexes: []IndexDescription{ + { + Name: "index1", + Fields: []IndexedFieldDescription{ + {Name: "test", Direction: Ascending}, + }, + }, + { + Name: "index2", + Fields: []IndexedFieldDescription{ + {Name: "test", Direction: Descending}, + }, + }, + }, + }, + field: "test", + expected: []IndexDescription{ + { + Name: "index1", + Fields: []IndexedFieldDescription{ + {Name: "test", Direction: Ascending}, + }, + }, + { + Name: "index2", + Fields: []IndexedFieldDescription{ + {Name: "test", Direction: Descending}, + }, + }, + }, + }, + { + name: "no indexes on field", + desc: CollectionDescription{ + Indexes: []IndexDescription{ + { + Name: "index1", + Fields: []IndexedFieldDescription{ + {Name: "other", Direction: Ascending}, + }, + }, + }, + }, + field: "test", + expected: []IndexDescription{}, + }, + { + name: "second field in composite index", + desc: CollectionDescription{ + Indexes: []IndexDescription{ + { + Name: "index1", + Fields: []IndexedFieldDescription{ + {Name: "other", Direction: Ascending}, + {Name: "test", Direction: Ascending}, + }, + }, + }, + }, + field: "test", + expected: []IndexDescription{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := tt.desc.CollectIndexesOnField(tt.field) + assert.Equal(t, tt.expected, actual) + }) + } +} diff --git a/planner/planner.go b/planner/planner.go index 5a87983947..d92106d877 100644 --- a/planner/planner.go +++ b/planner/planner.go @@ -342,18 +342,17 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan ) slct := node.subType.(*selectTopNode).selectNode desc := slct.collection.Description() - schema := slct.collection.Schema() - indexedFields := desc.CollectIndexedFields(&schema) - for _, indField := range indexedFields { - if ind, ok := filteredSubFields[indField.Name]; ok { + for subFieldName, subFieldInd := range filteredSubFields { + indexes := desc.CollectIndexesOnField(subFieldName) + if len(indexes) > 0 { subInd := node.documentMapping.FirstIndexOfName(node.subTypeName) relatedField := mapper.Field{Name: node.subTypeName, Index: subInd} fieldFilter := filter.UnwrapRelation(filter.CopyField( parentPlan.selectNode.filter, relatedField, - mapper.Field{Name: indField.Name, Index: ind}, + mapper.Field{Name: subFieldName, Index: subFieldInd}, ), relatedField) - err := node.invertJoinDirectionWithIndex(fieldFilter, indField) + err := node.invertJoinDirectionWithIndex(fieldFilter, indexes[0]) if err != nil { return err } diff --git a/planner/scan.go b/planner/scan.go index 3a44a812c9..c6417131c1 100644 --- a/planner/scan.go +++ b/planner/scan.go @@ -138,7 +138,7 @@ func (n *scanNode) tryAddField(fieldName string) bool { func (scan *scanNode) initFetcher( cid immutable.Option[string], - indexedField immutable.Option[client.FieldDescription], + index immutable.Option[client.IndexDescription], ) { var f fetcher.Fetcher if cid.HasValue() { @@ -146,13 +146,14 @@ func (scan *scanNode) initFetcher( } else { f = new(fetcher.DocumentFetcher) - if indexedField.HasValue() { - typeIndex := scan.documentMapping.FirstIndexOfName(indexedField.Value().Name) - field := mapper.Field{Index: typeIndex, Name: indexedField.Value().Name} + if index.HasValue() { + fieldName := index.Value().Fields[0].Name + typeIndex := scan.documentMapping.FirstIndexOfName(fieldName) + field := mapper.Field{Index: typeIndex, Name: fieldName} var indexFilter *mapper.Filter scan.filter, indexFilter = filter.SplitByFields(scan.filter, field) if indexFilter != nil { - fieldDesc, _ := scan.col.Schema().GetField(indexedField.Value().Name) + fieldDesc, _ := scan.col.Schema().GetField(fieldName) f = fetcher.NewIndexFetcher(f, fieldDesc, indexFilter) } } diff --git a/planner/select.go b/planner/select.go index 295ad3c475..fafc6f2788 100644 --- a/planner/select.go +++ b/planner/select.go @@ -296,20 +296,22 @@ func (n *selectNode) initSource() ([]aggregateNode, error) { return aggregates, nil } -func findFilteredByIndexedField(scanNode *scanNode) immutable.Option[client.FieldDescription] { +func findFilteredByIndexedField(scanNode *scanNode) immutable.Option[client.IndexDescription] { if scanNode.filter != nil { - schema := scanNode.col.Schema() - indexedFields := scanNode.col.Description().CollectIndexedFields(&schema) - for i := range indexedFields { - typeIndex := scanNode.documentMapping.FirstIndexOfName(indexedFields[i].Name) - if scanNode.filter.HasIndex(typeIndex) { - // we return the first found indexed field to keep it simple for now - // more sophisticated optimization logic can be added later - return immutable.Some(indexedFields[i]) + colDesc := scanNode.col.Description() + + for _, field := range scanNode.col.Schema().Fields { + if _, isFiltered := scanNode.filter.ExternalConditions[field.Name]; !isFiltered { + continue + } + indexes := colDesc.CollectIndexesOnField(field.Name) + if len(indexes) > 0 { + return immutable.Some(indexes[0]) } } } - return immutable.None[client.FieldDescription]() + + return immutable.None[client.IndexDescription]() } func (n *selectNode) initFields(selectReq *mapper.Select) ([]aggregateNode, error) { diff --git a/planner/type_join.go b/planner/type_join.go index 0d79b76ccd..9d9a27b969 100644 --- a/planner/type_join.go +++ b/planner/type_join.go @@ -606,12 +606,12 @@ func (join *invertibleTypeJoin) Next() (bool, error) { func (join *invertibleTypeJoin) invertJoinDirectionWithIndex( fieldFilter *mapper.Filter, - field client.FieldDescription, + index client.IndexDescription, ) error { subScan := getScanNode(join.subType) subScan.tryAddField(join.rootName + request.RelatedObjectID) subScan.filter = fieldFilter - subScan.initFetcher(immutable.Option[string]{}, immutable.Some(field)) + subScan.initFetcher(immutable.Option[string]{}, immutable.Some(index)) join.invert() diff --git a/tests/integration/index/query_with_index_combined_filter_test.go b/tests/integration/index/query_with_index_combined_filter_test.go index 98f8a216d6..eabc28067c 100644 --- a/tests/integration/index/query_with_index_combined_filter_test.go +++ b/tests/integration/index/query_with_index_combined_filter_test.go @@ -93,3 +93,40 @@ func TestQueryWithIndex_IfMultipleIndexFiltersWithRegular_ShouldFilter(t *testin testUtils.ExecuteTestCase(t, test) } + +func TestQueryWithIndex_FilterOnNonIndexedField_ShouldIgnoreIndex(t *testing.T) { + req := `query { + User(filter: { + age: {_eq: 44} + }) { + name + } + }` + test := testUtils.TestCase{ + Description: "If filter does not contain indexed field, index should be ignored", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String @index + age: Int + }`, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(0), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From d701adfa62b942ecfc8e71a1cd6ec26c7f441c85 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 Jan 2024 12:53:56 +0100 Subject: [PATCH 11/31] Make index fetcher use all fields of the index --- db/fetcher/indexer.go | 50 ++++++++++++++++++++++++++----------------- planner/scan.go | 3 +-- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/db/fetcher/indexer.go b/db/fetcher/indexer.go index b8608e2b7d..61a8da593b 100644 --- a/db/fetcher/indexer.go +++ b/db/fetcher/indexer.go @@ -30,7 +30,7 @@ type IndexFetcher struct { docFilter *mapper.Filter doc *encodedDocument mapping *core.DocumentMapping - indexedField client.FieldDescription + indexedFields []client.FieldDescription docFields []client.FieldDescription indexDesc client.IndexDescription indexIter indexIterator @@ -43,13 +43,13 @@ var _ Fetcher = (*IndexFetcher)(nil) // NewIndexFetcher creates a new IndexFetcher. func NewIndexFetcher( docFetcher Fetcher, - indexedFieldDesc client.FieldDescription, + indexDesc client.IndexDescription, indexFilter *mapper.Filter, ) *IndexFetcher { return &IndexFetcher{ - docFetcher: docFetcher, - indexedField: indexedFieldDesc, - indexFilter: indexFilter, + docFetcher: docFetcher, + indexDesc: indexDesc, + indexFilter: indexFilter, } } @@ -69,21 +69,27 @@ func (f *IndexFetcher) Init( f.mapping = docMapper f.txn = txn - for _, index := range col.Description().Indexes { - if index.Fields[0].Name == f.indexedField.Name { - f.indexDesc = index - f.indexDataStoreKey.IndexID = index.ID - break + f.indexDataStoreKey.IndexID = f.indexDesc.ID + f.indexDataStoreKey.CollectionID = f.col.ID() + + for _, indexedField := range f.indexDesc.Fields { + for _, field := range f.col.Schema().Fields { + if field.Name == indexedField.Name { + f.indexedFields = append(f.indexedFields, field) + break + } } } - f.indexDataStoreKey.CollectionID = f.col.ID() - + f.docFields = make([]client.FieldDescription, 0, len(fields)-len(f.indexedFields)) +outer: for i := range fields { - if fields[i].Name == f.indexedField.Name { - f.docFields = append(fields[:i], fields[i+1:]...) - break + for j := range f.indexedFields { + if fields[i].Name == f.indexedFields[j].Name { + continue outer + } } + f.docFields = append(f.docFields, fields[i]) } iter, err := createIndexIterator(f.indexDataStoreKey, f.indexFilter, &f.execInfo, f.indexDesc.Unique) @@ -123,17 +129,21 @@ func (f *IndexFetcher) FetchNext(ctx context.Context) (EncodedDocument, ExecInfo return nil, f.execInfo, nil } - property := &encProperty{ - Desc: f.indexedField, - Raw: res.key.FieldValues[0], + for i, indexedField := range f.indexedFields { + property := &encProperty{ + Desc: indexedField, + Raw: res.key.FieldValues[i], + } + + f.doc.properties[indexedField] = property } if f.indexDesc.Unique { f.doc.id = res.value } else { - f.doc.id = res.key.FieldValues[1] + f.doc.id = res.key.FieldValues[len(res.key.FieldValues)-1] } - f.doc.properties[f.indexedField] = property + f.execInfo.FieldsFetched++ if f.docFetcher != nil && len(f.docFields) > 0 { diff --git a/planner/scan.go b/planner/scan.go index c6417131c1..221a2cc244 100644 --- a/planner/scan.go +++ b/planner/scan.go @@ -153,8 +153,7 @@ func (scan *scanNode) initFetcher( var indexFilter *mapper.Filter scan.filter, indexFilter = filter.SplitByFields(scan.filter, field) if indexFilter != nil { - fieldDesc, _ := scan.col.Schema().GetField(fieldName) - f = fetcher.NewIndexFetcher(f, fieldDesc, indexFilter) + f = fetcher.NewIndexFetcher(f, index.Value(), indexFilter) } } From 1c6427b4373f0d6ede25fa150f8f3cb3d53a1c28 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 Jan 2024 15:14:25 +0100 Subject: [PATCH 12/31] Pass index and all filter conditions to index iter factory --- db/fetcher/indexer.go | 30 +++++++++++++----------------- db/fetcher/indexer_iterators.go | 9 +++++---- planner/scan.go | 11 +++++++---- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/db/fetcher/indexer.go b/db/fetcher/indexer.go index 61a8da593b..9aeb603c6f 100644 --- a/db/fetcher/indexer.go +++ b/db/fetcher/indexer.go @@ -23,19 +23,18 @@ import ( // IndexFetcher is a fetcher that fetches documents by index. // It fetches only the indexed field and the rest of the fields are fetched by the internal fetcher. type IndexFetcher struct { - docFetcher Fetcher - col client.Collection - txn datastore.Txn - indexFilter *mapper.Filter - docFilter *mapper.Filter - doc *encodedDocument - mapping *core.DocumentMapping - indexedFields []client.FieldDescription - docFields []client.FieldDescription - indexDesc client.IndexDescription - indexIter indexIterator - indexDataStoreKey core.IndexDataStoreKey - execInfo ExecInfo + docFetcher Fetcher + col client.Collection + txn datastore.Txn + indexFilter *mapper.Filter + docFilter *mapper.Filter + doc *encodedDocument + mapping *core.DocumentMapping + indexedFields []client.FieldDescription + docFields []client.FieldDescription + indexDesc client.IndexDescription + indexIter indexIterator + execInfo ExecInfo } var _ Fetcher = (*IndexFetcher)(nil) @@ -69,9 +68,6 @@ func (f *IndexFetcher) Init( f.mapping = docMapper f.txn = txn - f.indexDataStoreKey.IndexID = f.indexDesc.ID - f.indexDataStoreKey.CollectionID = f.col.ID() - for _, indexedField := range f.indexDesc.Fields { for _, field := range f.col.Schema().Fields { if field.Name == indexedField.Name { @@ -92,7 +88,7 @@ outer: f.docFields = append(f.docFields, fields[i]) } - iter, err := createIndexIterator(f.indexDataStoreKey, f.indexFilter, &f.execInfo, f.indexDesc.Unique) + iter, err := createIndexIterator(f.indexFilter, &f.execInfo, f.indexDesc, f.col.ID()) if err != nil { return err } diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index 1e896e1b3b..bbd167ba08 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -396,11 +396,12 @@ func (m *indexLikeMatcher) doesMatch(currentVal string) bool { } func createIndexIterator( - indexDataStoreKey core.IndexDataStoreKey, indexFilterConditions *mapper.Filter, execInfo *ExecInfo, - isUnique bool, + indexDesc client.IndexDescription, + colID uint32, ) (indexIterator, error) { + indexDataStoreKey := core.IndexDataStoreKey{CollectionID: colID, IndexID: indexDesc.ID} var op string var filterVal any for _, indexFilterCond := range indexFilterConditions.Conditions { @@ -425,7 +426,7 @@ func createIndexIterator( switch op { case opEq: - if isUnique { + if indexDesc.Unique { return &eqSingleIndexIterator{ indexKey: indexDataStoreKey, filterValueHolder: filterValueHolder{ @@ -503,7 +504,7 @@ func createIndexIterator( } if op == opIn { var iter filterValueIndexIterator - if isUnique { + if indexDesc.Unique { iter = &eqSingleIndexIterator{ indexKey: indexDataStoreKey, execInfo: execInfo, diff --git a/planner/scan.go b/planner/scan.go index 221a2cc244..a11f18de5e 100644 --- a/planner/scan.go +++ b/planner/scan.go @@ -147,11 +147,14 @@ func (scan *scanNode) initFetcher( f = new(fetcher.DocumentFetcher) if index.HasValue() { - fieldName := index.Value().Fields[0].Name - typeIndex := scan.documentMapping.FirstIndexOfName(fieldName) - field := mapper.Field{Index: typeIndex, Name: fieldName} + fields := make([]mapper.Field, 0, len(index.Value().Fields)) + for _, field := range index.Value().Fields { + fieldName := field.Name + typeIndex := scan.documentMapping.FirstIndexOfName(fieldName) + fields = append(fields, mapper.Field{Index: typeIndex, Name: fieldName}) + } var indexFilter *mapper.Filter - scan.filter, indexFilter = filter.SplitByFields(scan.filter, field) + scan.filter, indexFilter = filter.SplitByFields(scan.filter, fields...) if indexFilter != nil { f = fetcher.NewIndexFetcher(f, index.Value(), indexFilter) } From c40fb39d6065d95eab194f63038b504feb4bf7b7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 4 Jan 2024 11:15:32 +0100 Subject: [PATCH 13/31] Fix edge-case in filtering algorithm --- planner/filter/copy_field.go | 3 ++- planner/filter/copy_field_test.go | 19 +++++++++++++++++-- planner/filter/split.go | 3 +++ planner/filter/split_test.go | 19 +++++++++++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/planner/filter/copy_field.go b/planner/filter/copy_field.go index 70b5dc2956..c824e9a9d4 100644 --- a/planner/filter/copy_field.go +++ b/planner/filter/copy_field.go @@ -15,7 +15,8 @@ import ( ) // CopyField copies the given field from the provided filter. -// Multiple fields can be passed to copy related objects with a certain field. +// Multiple fields can be passed to copy related objects with a certain field. +// In this case every subsequent field is a sub field of the previous one. Eg. bool.author.name // The result filter preserves the structure of the original filter. func CopyField(filter *mapper.Filter, fields ...mapper.Field) *mapper.Filter { if filter == nil || len(fields) == 0 { diff --git a/planner/filter/copy_field_test.go b/planner/filter/copy_field_test.go index 1714db55b6..611f1d1fd8 100644 --- a/planner/filter/copy_field_test.go +++ b/planner/filter/copy_field_test.go @@ -120,12 +120,12 @@ func TestCopyField(t *testing.T) { } } -func TestCopyFieldOfNullFilter(t *testing.T) { +func TestCopyField_IfFilterIsNil_NoOp(t *testing.T) { actualFilter := CopyField(nil, mapper.Field{Index: 1}) assert.Nil(t, actualFilter) } -func TestCopyFieldWithNoFieldGiven(t *testing.T) { +func TestCopyField_IfNoFieldGiven_NoOp(t *testing.T) { filter := mapper.NewFilter() filter.Conditions = map[connor.FilterKey]any{ &mapper.PropertyIndex{Index: 0}: &mapper.Operator{Operation: "_eq"}, @@ -133,3 +133,18 @@ func TestCopyFieldWithNoFieldGiven(t *testing.T) { actualFilter := CopyField(filter) assert.Nil(t, actualFilter) } + +func TestCopyField_IfSecondFieldIsNotSubField_NoOp(t *testing.T) { + mapping := getDocMapping() + inputFilter := mapper.ToFilter(request.Filter{Conditions: map[string]any{ + "name": m("_eq", "John"), + "age": m("_gt", 55), + }}, mapping) + + var actualFilter *mapper.Filter + assert.NotPanics(t, func() { + actualFilter = CopyField(inputFilter, mapper.Field{Index: authorNameInd}, mapper.Field{Index: 666}) + }) + + assert.Nil(t, actualFilter) +} diff --git a/planner/filter/split.go b/planner/filter/split.go index f7ff705b68..e562c8165a 100644 --- a/planner/filter/split.go +++ b/planner/filter/split.go @@ -38,6 +38,9 @@ func SplitByFields(filter *mapper.Filter, fields ...mapper.Field) (*mapper.Filte for _, field := range fields[1:] { newSplitF := CopyField(filter, field) + if newSplitF == nil { + continue + } splitF.Conditions = Merge(splitF.Conditions, newSplitF.Conditions) RemoveField(filter, field) } diff --git a/planner/filter/split_test.go b/planner/filter/split_test.go index 9ed3700f22..61197a0680 100644 --- a/planner/filter/split_test.go +++ b/planner/filter/split_test.go @@ -70,6 +70,25 @@ func TestSplitFilter(t *testing.T) { "verified": m("_eq", false), }, }, + { + name: "split by fields that are not present", + inputFilter: map[string]any{ + "name": m("_eq", "John"), + "age": m("_gt", 55), + "verified": m("_eq", false), + }, + inputFields: []mapper.Field{ + {Index: authorNameInd}, + {Index: 100}, + {Index: authorAgeInd}, + {Index: 200}, + }, + expectedFilter1: m("verified", m("_eq", false)), + expectedFilter2: map[string]any{ + "name": m("_eq", "John"), + "age": m("_gt", 55), + }, + }, } mapping := getDocMapping() From 8bf56586faf10f0da899657c0cf712558c441ba2 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 5 Jan 2024 11:09:56 +0100 Subject: [PATCH 14/31] Extract matcher creation into another function --- db/fetcher/indexer.go | 2 +- db/fetcher/indexer_iterators.go | 207 ++++++++++++++------------------ 2 files changed, 94 insertions(+), 115 deletions(-) diff --git a/db/fetcher/indexer.go b/db/fetcher/indexer.go index 9aeb603c6f..1b01f884c0 100644 --- a/db/fetcher/indexer.go +++ b/db/fetcher/indexer.go @@ -88,7 +88,7 @@ outer: f.docFields = append(f.docFields, fields[i]) } - iter, err := createIndexIterator(f.indexFilter, &f.execInfo, f.indexDesc, f.col.ID()) + iter, err := f.createIndexIterator() if err != nil { return err } diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index bbd167ba08..79758bd8cb 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -165,16 +165,6 @@ type inIndexIterator struct { hasIterator bool } -func newInIndexIterator( - indexIter filterValueIndexIterator, - filterValues [][]byte, -) *inIndexIterator { - return &inIndexIterator{ - filterValueIndexIterator: indexIter, - filterValues: filterValues, - } -} - func (i *inIndexIterator) nextIterator() (bool, error) { if i.nextValIndex > 0 { err := i.filterValueIndexIterator.Close() @@ -309,15 +299,6 @@ func (m *indexByteValuesMatcher) Match(value []byte) (bool, error) { return m.evalFunc(res), nil } -// matcher if _ne condition is met -type neIndexMatcher struct { - value []byte -} - -func (m *neIndexMatcher) Match(value []byte) (bool, error) { - return !bytes.Equal(value, m.value), nil -} - // checks if the index value is or is not in the given array type indexInArrayMatcher struct { values map[string]bool @@ -395,16 +376,67 @@ func (m *indexLikeMatcher) doesMatch(currentVal string) bool { } } -func createIndexIterator( - indexFilterConditions *mapper.Filter, - execInfo *ExecInfo, - indexDesc client.IndexDescription, - colID uint32, -) (indexIterator, error) { - indexDataStoreKey := core.IndexDataStoreKey{CollectionID: colID, IndexID: indexDesc.ID} +func createValueMatcher(op string, filterVal any) (valueMatcher, error) { + switch op { + case opEq, opGt, opGe, opLt, opLe, opNe: + fieldValue := client.NewFieldValue(client.LWW_REGISTER, filterVal) + + valueBytes, err := fieldValue.Bytes() + if err != nil { + return nil, err + } + + m := &indexByteValuesMatcher{value: valueBytes} + switch op { + case opEq: + m.evalFunc = func(res int) bool { return res == 0 } + case opGt: + m.evalFunc = func(res int) bool { return res > 0 } + case opGe: + m.evalFunc = func(res int) bool { return res > 0 || res == 0 } + case opLt: + m.evalFunc = func(res int) bool { return res < 0 } + case opLe: + m.evalFunc = func(res int) bool { return res < 0 || res == 0 } + case opNe: + m.evalFunc = func(res int) bool { return res != 0 } + } + return m, nil + case opIn, opNin: + inArr, ok := filterVal.([]any) + if !ok { + return nil, errors.New("invalid _in/_nin value") + } + valArr := make([][]byte, 0, len(inArr)) + for _, v := range inArr { + fieldValue := client.NewFieldValue(client.LWW_REGISTER, v) + valueBytes, err := fieldValue.Bytes() + if err != nil { + return nil, err + } + valArr = append(valArr, valueBytes) + } + return newNinIndexCmp(valArr, op == opIn), nil + case opLike, opNlike: + return newLikeIndexCmp(filterVal.(string), op == opLike), nil + } + + return nil, errors.New("invalid index filter condition") +} + +func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { var op string var filterVal any - for _, indexFilterCond := range indexFilterConditions.Conditions { + for filterKey, indexFilterCond := range f.indexFilter.Conditions { + propKey, ok := filterKey.(*mapper.PropertyIndex) + if !ok { + continue + } + fieldInd := f.mapping.FirstIndexOfName(f.indexedFields[0].Name) + if fieldInd != propKey.Index { + continue + } + condMap := indexFilterCond.(map[connor.FilterKey]any) var key connor.FilterKey for key, filterVal = range condMap { @@ -415,125 +447,72 @@ func createIndexIterator( break } + indexDataStoreKey := core.IndexDataStoreKey{CollectionID: f.col.ID(), IndexID: f.indexDesc.ID} switch op { - case opEq, opGt, opGe, opLt, opLe, opNe: - fieldValue := client.NewFieldValue(client.LWW_REGISTER, filterVal) + case opEq: + writableValue := client.NewCBORValue(client.LWW_REGISTER, filterVal) - valueBytes, err := fieldValue.Bytes() + valueBytes, err := writableValue.Bytes() if err != nil { return nil, err } - switch op { - case opEq: - if indexDesc.Unique { - return &eqSingleIndexIterator{ - indexKey: indexDataStoreKey, - filterValueHolder: filterValueHolder{ - value: valueBytes, - }, - execInfo: execInfo, - }, nil - } else { - return &eqPrefixIndexIterator{ - indexKey: indexDataStoreKey, - filterValueHolder: filterValueHolder{ - value: valueBytes, - }, - execInfo: execInfo, - }, nil - } - case opGt: - return &scanningIndexIterator{ - indexKey: indexDataStoreKey, - matcher: &indexByteValuesMatcher{ - value: valueBytes, - evalFunc: func(res int) bool { return res > 0 }, - }, - execInfo: execInfo, - }, nil - case opGe: - return &scanningIndexIterator{ + if f.indexDesc.Unique { + return &eqSingleIndexIterator{ indexKey: indexDataStoreKey, - matcher: &indexByteValuesMatcher{ - value: valueBytes, - evalFunc: func(res int) bool { return res > 0 || res == 0 }, - }, - execInfo: execInfo, - }, nil - case opLt: - return &scanningIndexIterator{ - indexKey: indexDataStoreKey, - matcher: &indexByteValuesMatcher{ - value: valueBytes, - evalFunc: func(res int) bool { return res < 0 }, - }, - execInfo: execInfo, - }, nil - case opLe: - return &scanningIndexIterator{ - indexKey: indexDataStoreKey, - matcher: &indexByteValuesMatcher{ - value: valueBytes, - evalFunc: func(res int) bool { return res < 0 || res == 0 }, + filterValueHolder: filterValueHolder{ + value: valueBytes, }, - execInfo: execInfo, + execInfo: &f.execInfo, }, nil - case opNe: - return &scanningIndexIterator{ + } else { + return &eqPrefixIndexIterator{ indexKey: indexDataStoreKey, - matcher: &neIndexMatcher{ + filterValueHolder: filterValueHolder{ value: valueBytes, }, - execInfo: execInfo, + execInfo: &f.execInfo, }, nil } - case opIn, opNin: + case opIn: inArr, ok := filterVal.([]any) if !ok { return nil, errors.New("invalid _in/_nin value") } valArr := make([][]byte, 0, len(inArr)) for _, v := range inArr { - fieldValue := client.NewFieldValue(client.LWW_REGISTER, v) - valueBytes, err := fieldValue.Bytes() + writableValue := client.NewCBORValue(client.LWW_REGISTER, v) + valueBytes, err := writableValue.Bytes() if err != nil { return nil, err } valArr = append(valArr, valueBytes) } - if op == opIn { - var iter filterValueIndexIterator - if indexDesc.Unique { - iter = &eqSingleIndexIterator{ - indexKey: indexDataStoreKey, - execInfo: execInfo, - } - } else { - iter = &eqPrefixIndexIterator{ - indexKey: indexDataStoreKey, - execInfo: execInfo, - } + var iter filterValueIndexIterator + if f.indexDesc.Unique { + iter = &eqSingleIndexIterator{ + indexKey: indexDataStoreKey, + execInfo: &f.execInfo, } - return newInIndexIterator(iter, valArr), nil } else { - return &scanningIndexIterator{ + iter = &eqPrefixIndexIterator{ indexKey: indexDataStoreKey, - matcher: newNinIndexCmp(valArr, false), - execInfo: execInfo, - }, nil + execInfo: &f.execInfo, + } } - case opLike: - return &scanningIndexIterator{ - indexKey: indexDataStoreKey, - matcher: newLikeIndexCmp(filterVal.(string), true), - execInfo: execInfo, + return &inIndexIterator{ + filterValueIndexIterator: iter, + filterValues: valArr, }, nil - case opNlike: + case opGt, opGe, opLt, opLe, opNe, opNin, opLike, opNlike: + m, err := createValueMatcher(op, filterVal) + if err != nil { + return nil, err + } return &scanningIndexIterator{ indexKey: indexDataStoreKey, - matcher: newLikeIndexCmp(filterVal.(string), false), - execInfo: execInfo, + matcher: m, + execInfo: &f.execInfo, }, nil } From b87b9ae3921dd02c929cbae895bc2929953085f9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 5 Jan 2024 16:53:40 +0100 Subject: [PATCH 15/31] Add multiple matchers --- db/fetcher/indexer_iterators.go | 38 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index 79758bd8cb..a46b72cfab 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -217,37 +217,34 @@ func (i *inIndexIterator) Close() error { } type errorCheckingFilter struct { - matcher valueMatcher - err error + matchers []valueMatcher + err error + execInfo *ExecInfo } func (f *errorCheckingFilter) Filter(e query.Entry) bool { if f.err != nil { return false } + f.execInfo.IndexesFetched++ + indexKey, err := core.NewIndexDataStoreKey(e.Key) if err != nil { f.err = err return false } - res, err := f.matcher.Match(indexKey.FieldValues[0]) - if err != nil { - f.err = err - return false - } - return res -} -// execInfoIndexMatcherDecorator is a decorator for indexMatcher that counts the number -// of indexes fetched on every call to Match. -type execInfoIndexMatcherDecorator struct { - matcher valueMatcher - execInfo *ExecInfo -} - -func (d *execInfoIndexMatcherDecorator) Match(value []byte) (bool, error) { - d.execInfo.IndexesFetched++ - return d.matcher.Match(value) + for i := range f.matchers { + res, err := f.matchers[i].Match(indexKey.FieldValues[i]) + if err != nil { + f.err = err + return false + } + if !res { + return false + } + } + return true } type scanningIndexIterator struct { @@ -259,7 +256,8 @@ type scanningIndexIterator struct { } func (i *scanningIndexIterator) Init(ctx context.Context, store datastore.DSReaderWriter) error { - i.filter.matcher = &execInfoIndexMatcherDecorator{matcher: i.matcher, execInfo: i.execInfo} + i.filter.matchers = []valueMatcher{i.matcher} + i.filter.execInfo = i.execInfo iter, err := store.Query(ctx, query.Query{ Prefix: i.indexKey.ToString(), From 990aa8e647c453a18c2e5a076a22fe92fbb0aca8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 Jan 2024 11:20:10 +0100 Subject: [PATCH 16/31] Create multiple value matchers --- db/fetcher/indexer_iterators.go | 199 +++++++++++++++++++------------- 1 file changed, 118 insertions(+), 81 deletions(-) diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index a46b72cfab..03740dda94 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -39,6 +39,7 @@ const ( opNin = "_nin" opLike = "_like" opNlike = "_nlike" + opAny = "_any" ) // indexIterator is an iterator over index keys. @@ -80,15 +81,20 @@ func (i *queryResultIterator) Close() error { } type eqPrefixIndexIterator struct { - filterValueHolder - indexKey core.IndexDataStoreKey - execInfo *ExecInfo + indexKey core.IndexDataStoreKey + keyFieldValue []byte + execInfo *ExecInfo + matchers []valueMatcher queryResultIterator } +func (i *eqPrefixIndexIterator) SetKeyFieldValue(value []byte) { + i.keyFieldValue = value +} + func (i *eqPrefixIndexIterator) Init(ctx context.Context, store datastore.DSReaderWriter) error { - i.indexKey.FieldValues = [][]byte{i.value} + i.indexKey.FieldValues = [][]byte{i.keyFieldValue} resultIter, err := store.Query(ctx, query.Query{ Prefix: i.indexKey.ToString(), }) @@ -107,28 +113,24 @@ func (i *eqPrefixIndexIterator) Next() (indexIterResult, error) { return res, err } -type filterValueIndexIterator interface { +type keyFieldIndexIterator interface { indexIterator - SetFilterValue([]byte) -} - -type filterValueHolder struct { - value []byte -} - -func (h *filterValueHolder) SetFilterValue(value []byte) { - h.value = value + SetKeyFieldValue([]byte) } type eqSingleIndexIterator struct { - filterValueHolder - indexKey core.IndexDataStoreKey - execInfo *ExecInfo + indexKey core.IndexDataStoreKey + keyFieldValue []byte + execInfo *ExecInfo ctx context.Context store datastore.DSReaderWriter } +func (i *eqSingleIndexIterator) SetKeyFieldValue(value []byte) { + i.keyFieldValue = value +} + func (i *eqSingleIndexIterator) Init(ctx context.Context, store datastore.DSReaderWriter) error { i.ctx = ctx i.store = store @@ -139,7 +141,7 @@ func (i *eqSingleIndexIterator) Next() (indexIterResult, error) { if i.store == nil { return indexIterResult{}, nil } - i.indexKey.FieldValues = [][]byte{i.value} + i.indexKey.FieldValues = [][]byte{i.keyFieldValue} val, err := i.store.Get(i.ctx, i.indexKey.ToDS()) if err != nil { if errors.Is(err, ds.ErrNotFound) { @@ -157,28 +159,28 @@ func (i *eqSingleIndexIterator) Close() error { } type inIndexIterator struct { - filterValueIndexIterator - filterValues [][]byte - nextValIndex int - ctx context.Context - store datastore.DSReaderWriter - hasIterator bool + keyFieldIndexIterator + keyFieldValues [][]byte + nextValIndex int + ctx context.Context + store datastore.DSReaderWriter + hasIterator bool } func (i *inIndexIterator) nextIterator() (bool, error) { if i.nextValIndex > 0 { - err := i.filterValueIndexIterator.Close() + err := i.keyFieldIndexIterator.Close() if err != nil { return false, err } } - if i.nextValIndex >= len(i.filterValues) { + if i.nextValIndex >= len(i.keyFieldValues) { return false, nil } - i.SetFilterValue(i.filterValues[i.nextValIndex]) - err := i.filterValueIndexIterator.Init(i.ctx, i.store) + i.SetKeyFieldValue(i.keyFieldValues[i.nextValIndex]) + err := i.keyFieldIndexIterator.Init(i.ctx, i.store) if err != nil { return false, err } @@ -196,7 +198,7 @@ func (i *inIndexIterator) Init(ctx context.Context, store datastore.DSReaderWrit func (i *inIndexIterator) Next() (indexIterResult, error) { for i.hasIterator { - res, err := i.filterValueIndexIterator.Next() + res, err := i.keyFieldIndexIterator.Next() if err != nil { return indexIterResult{}, err } @@ -222,6 +224,19 @@ type errorCheckingFilter struct { execInfo *ExecInfo } +func executeValueMatchers(matchers []valueMatcher, values [][]byte) (bool, error) { + for i := range matchers { + res, err := matchers[i].Match(values[i]) + if err != nil { + return false, err + } + if !res { + return false, nil + } + } + return true, nil +} + func (f *errorCheckingFilter) Filter(e query.Entry) bool { if f.err != nil { return false @@ -234,29 +249,21 @@ func (f *errorCheckingFilter) Filter(e query.Entry) bool { return false } - for i := range f.matchers { - res, err := f.matchers[i].Match(indexKey.FieldValues[i]) - if err != nil { - f.err = err - return false - } - if !res { - return false - } - } - return true + var res bool + res, f.err = executeValueMatchers(f.matchers, indexKey.FieldValues) + return res } type scanningIndexIterator struct { queryResultIterator indexKey core.IndexDataStoreKey - matcher valueMatcher + matchers []valueMatcher filter errorCheckingFilter execInfo *ExecInfo } func (i *scanningIndexIterator) Init(ctx context.Context, store datastore.DSReaderWriter) error { - i.filter.matchers = []valueMatcher{i.matcher} + i.filter.matchers = i.matchers i.filter.execInfo = i.execInfo iter, err := store.Query(ctx, query.Query{ @@ -374,6 +381,10 @@ func (m *indexLikeMatcher) doesMatch(currentVal string) bool { } } +type anyMatcher struct{} + +func (m *anyMatcher) Match([]byte) (bool, error) { return true, nil } + func createValueMatcher(op string, filterVal any) (valueMatcher, error) { switch op { case opEq, opGt, opGe, opLt, opLe, opNe: @@ -417,76 +428,102 @@ func createValueMatcher(op string, filterVal any) (valueMatcher, error) { return newNinIndexCmp(valArr, op == opIn), nil case opLike, opNlike: return newLikeIndexCmp(filterVal.(string), op == opLike), nil + case opAny: + return &anyMatcher{}, nil } return nil, errors.New("invalid index filter condition") } -func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { - var op string - var filterVal any - for filterKey, indexFilterCond := range f.indexFilter.Conditions { - propKey, ok := filterKey.(*mapper.PropertyIndex) - if !ok { - continue - } - fieldInd := f.mapping.FirstIndexOfName(f.indexedFields[0].Name) - if fieldInd != propKey.Index { - continue +func createValueMatchers(conditions []fieldFilterCond) ([]valueMatcher, error) { + matchers := make([]valueMatcher, 0, len(conditions)) + for i := range conditions { + m, err := createValueMatcher(conditions[i].op, conditions[i].val) + if err != nil { + return nil, err } + matchers = append(matchers, m) + } + return matchers, nil +} + +type fieldFilterCond struct { + op string + val any +} + +func (f *IndexFetcher) determineFieldFilterConditions() []fieldFilterCond { + result := make([]fieldFilterCond, 0, len(f.indexedFields)) + for i := range f.indexedFields { + fieldInd := f.mapping.FirstIndexOfName(f.indexedFields[i].Name) + found := false + for filterKey, indexFilterCond := range f.indexFilter.Conditions { + propKey, ok := filterKey.(*mapper.PropertyIndex) + if !ok { + continue + } + if fieldInd != propKey.Index { + continue + } - condMap := indexFilterCond.(map[connor.FilterKey]any) - var key connor.FilterKey - for key, filterVal = range condMap { + found = true + + condMap := indexFilterCond.(map[connor.FilterKey]any) + for key, filterVal := range condMap { + opKey := key.(*mapper.Operator) + result = append(result, fieldFilterCond{op: opKey.Operation, val: filterVal}) + break + } break } - opKey := key.(*mapper.Operator) - op = opKey.Operation - break + if !found { + result = append(result, fieldFilterCond{op: opAny}) + } } + return result +} +func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { + fieldConditions := f.determineFieldFilterConditions() indexDataStoreKey := core.IndexDataStoreKey{CollectionID: f.col.ID(), IndexID: f.indexDesc.ID} - switch op { + + switch fieldConditions[0].op { case opEq: - writableValue := client.NewCBORValue(client.LWW_REGISTER, filterVal) + writableValue := client.NewCBORValue(client.LWW_REGISTER, fieldConditions[0].val) - valueBytes, err := writableValue.Bytes() + keyValueBytes, err := writableValue.Bytes() if err != nil { return nil, err } if f.indexDesc.Unique { return &eqSingleIndexIterator{ - indexKey: indexDataStoreKey, - filterValueHolder: filterValueHolder{ - value: valueBytes, - }, - execInfo: &f.execInfo, + indexKey: indexDataStoreKey, + keyFieldValue: keyValueBytes, + execInfo: &f.execInfo, }, nil } else { return &eqPrefixIndexIterator{ - indexKey: indexDataStoreKey, - filterValueHolder: filterValueHolder{ - value: valueBytes, - }, - execInfo: &f.execInfo, + indexKey: indexDataStoreKey, + keyFieldValue: keyValueBytes, + execInfo: &f.execInfo, }, nil } case opIn: - inArr, ok := filterVal.([]any) + inArr, ok := fieldConditions[0].val.([]any) if !ok { return nil, errors.New("invalid _in/_nin value") } - valArr := make([][]byte, 0, len(inArr)) + keyFieldArr := make([][]byte, 0, len(inArr)) for _, v := range inArr { writableValue := client.NewCBORValue(client.LWW_REGISTER, v) - valueBytes, err := writableValue.Bytes() + keyFieldBytes, err := writableValue.Bytes() if err != nil { return nil, err } - valArr = append(valArr, valueBytes) + keyFieldArr = append(keyFieldArr, keyFieldBytes) } - var iter filterValueIndexIterator + var iter keyFieldIndexIterator if f.indexDesc.Unique { iter = &eqSingleIndexIterator{ indexKey: indexDataStoreKey, @@ -499,17 +536,17 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { } } return &inIndexIterator{ - filterValueIndexIterator: iter, - filterValues: valArr, + keyFieldIndexIterator: iter, + keyFieldValues: keyFieldArr, }, nil case opGt, opGe, opLt, opLe, opNe, opNin, opLike, opNlike: - m, err := createValueMatcher(op, filterVal) + matchers, err := createValueMatchers(fieldConditions) if err != nil { return nil, err } return &scanningIndexIterator{ indexKey: indexDataStoreKey, - matcher: m, + matchers: matchers, execInfo: &f.execInfo, }, nil } From 86a6ee76842c77b02aa9cc08b83e1f65a164fbde Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 Jan 2024 21:36:52 +0100 Subject: [PATCH 17/31] Fix edge-case with filter normalization --- planner/filter/normalize.go | 7 ++++++- planner/filter/split_test.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/planner/filter/normalize.go b/planner/filter/normalize.go index 181b1f8485..65317f2170 100644 --- a/planner/filter/normalize.go +++ b/planner/filter/normalize.go @@ -185,7 +185,12 @@ func normalizeProperties(parentKey connor.FilterKey, conditions []any) []any { // if canMergeAnd is true, all _and groups will be merged props := make(map[int][]any) for _, c := range conditions { - for key, val := range c.(map[connor.FilterKey]any) { + cMap, ok := c.(map[connor.FilterKey]any) + if !ok { + result = append(result, c) + continue + } + for key, val := range cMap { op, ok := key.(*mapper.Operator) if canMergeAnd && ok && op.Operation == request.FilterOpAnd { merge = append(merge, val.([]any)...) diff --git a/planner/filter/split_test.go b/planner/filter/split_test.go index 61197a0680..221bd31527 100644 --- a/planner/filter/split_test.go +++ b/planner/filter/split_test.go @@ -89,6 +89,22 @@ func TestSplitFilter(t *testing.T) { "age": m("_gt", 55), }, }, + { + name: "filter with two []any slices", + inputFilter: map[string]any{ + "age": m("_in", []any{10, 20, 30}), + "name": m("_in", []any{"John", "Bob"}), + }, + inputFields: []mapper.Field{ + {Index: authorNameInd}, + {Index: authorAgeInd}, + }, + expectedFilter1: nil, + expectedFilter2: map[string]any{ + "age": m("_in", []any{10, 20, 30}), + "name": m("_in", []any{"John", "Bob"}), + }, + }, } mapping := getDocMapping() From cb924ef1fa54e2424befea754e25d05fa5cd1cb4 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 Jan 2024 23:09:11 +0100 Subject: [PATCH 18/31] Execute matching for remaining fields of index --- db/fetcher/indexer_iterators.go | 36 ++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index 03740dda94..d181d7e82c 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -106,11 +106,21 @@ func (i *eqPrefixIndexIterator) Init(ctx context.Context, store datastore.DSRead } func (i *eqPrefixIndexIterator) Next() (indexIterResult, error) { - res, err := i.queryResultIterator.Next() - if res.foundKey { + for { + res, err := i.queryResultIterator.Next() + if err != nil || !res.foundKey { + return res, err + } i.execInfo.IndexesFetched++ + doesMatch, err := executeValueMatchers(i.matchers, res.key.FieldValues) + if err != nil { + return indexIterResult{}, err + } + if !doesMatch { + continue + } + return res, err } - return res, err } type keyFieldIndexIterator interface { @@ -487,6 +497,11 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { fieldConditions := f.determineFieldFilterConditions() indexDataStoreKey := core.IndexDataStoreKey{CollectionID: f.col.ID(), IndexID: f.indexDesc.ID} + matchers, err := createValueMatchers(fieldConditions) + if err != nil { + return nil, err + } + switch fieldConditions[0].op { case opEq: writableValue := client.NewCBORValue(client.LWW_REGISTER, fieldConditions[0].val) @@ -496,6 +511,10 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { return nil, err } + if len(matchers) > 1 { + matchers[0] = &anyMatcher{} + } + if f.indexDesc.Unique { return &eqSingleIndexIterator{ indexKey: indexDataStoreKey, @@ -507,6 +526,7 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { indexKey: indexDataStoreKey, keyFieldValue: keyValueBytes, execInfo: &f.execInfo, + matchers: matchers, }, nil } case opIn: @@ -523,6 +543,11 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { } keyFieldArr = append(keyFieldArr, keyFieldBytes) } + + if len(matchers) > 1 { + matchers[0] = &anyMatcher{} + } + var iter keyFieldIndexIterator if f.indexDesc.Unique { iter = &eqSingleIndexIterator{ @@ -533,6 +558,7 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { iter = &eqPrefixIndexIterator{ indexKey: indexDataStoreKey, execInfo: &f.execInfo, + matchers: matchers, } } return &inIndexIterator{ @@ -540,10 +566,6 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { keyFieldValues: keyFieldArr, }, nil case opGt, opGe, opLt, opLe, opNe, opNin, opLike, opNlike: - matchers, err := createValueMatchers(fieldConditions) - if err != nil { - return nil, err - } return &scanningIndexIterator{ indexKey: indexDataStoreKey, matchers: matchers, From 03f36313aa3282de46c7bbc261cc540af397e950 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 8 Jan 2024 23:09:19 +0100 Subject: [PATCH 19/31] Add tests for composite index --- ...y_with_composite_index_only_filter_test.go | 619 ++++++++++++++++++ 1 file changed, 619 insertions(+) create mode 100644 tests/integration/index/query_with_composite_index_only_filter_test.go diff --git a/tests/integration/index/query_with_composite_index_only_filter_test.go b/tests/integration/index/query_with_composite_index_only_filter_test.go new file mode 100644 index 0000000000..e031b3f6ff --- /dev/null +++ b/tests/integration/index/query_with_composite_index_only_filter_test.go @@ -0,0 +1,619 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +const schemaWithNameAgeIndex = ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }` + +const schemaWithAgeNameIndex = ` + type User @index(fields: ["age", "name"]) { + name: String + age: Int + email: String + }` + +func TestQueryWithCompositeIndex_WithEqualFilter_ShouldFetch(t *testing.T) { + req1 := `query { + User(filter: {name: {_eq: "Islam"}}) { + name + age + } + }` + req2 := `query { + User(filter: {name: {_eq: "Islam"}, age: {_eq: 32}}) { + name + age + } + }` + req3 := `query { + User(filter: {name: {_eq: "Islam"}, age: {_eq: 66}}) { + name + age + } + }` + test := testUtils.TestCase{ + Description: "Test filtering on composite index with _eq filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req1, + Results: []map[string]any{ + {"name": "Islam", "age": 32}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(1), + }, + testUtils.Request{ + Request: req2, + Results: []map[string]any{ + {"name": "Islam", "age": 32}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(1), + }, + testUtils.Request{ + Request: req3, + Results: []map[string]any{}, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithGreaterThanFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Keenan"}, age: {_gt: 44}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _gt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithGreaterThanFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Keenan"}, age: {_gt: 44}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _gt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithGreaterOrEqualFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Keenan"}, age: {_ge: 44},}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _ge filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithGreaterOrEqualFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_ge: 44}, name: {_ne: "Keenan"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _ge filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithLessThanFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_lt: 24}, name: {_ne: "Shahzad"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _lt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Bruno"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithLessThanFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_lt: 24}, name: {_ne: "Shahzad"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _lt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Bruno"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithLessOrEqualFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_le: 28}, name: {_ne: "Bruno"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _le filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Shahzad"}, + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithLessOrEqualFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_le: 28}, name: {_ne: "Bruno"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _le filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Fred"}, + {"name": "Shahzad"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithNotEqualFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Islam"}, age: {_ne: 28}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _ne filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Addo"}, + {"name": "Andy"}, + {"name": "John"}, + {"name": "Bruno"}, + {"name": "Chris"}, + {"name": "Keenan"}, + {"name": "Shahzad"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(8).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithInFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_in: [20, 28, 33]}, name: {_in: ["Addo", "Andy", "Fred"]}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _in filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Andy"}, + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(3), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithNotInFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_nin: [20, 23, 28, 42]}, name: {_nin: ["John", "Andy", "Chris"]}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _nin filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Islam"}, + {"name": "Keenan"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithLikeFilter_ShouldFetch(t *testing.T) { + req1 := `query { + User(filter: {email: {_like: "a%"}, name: {_like: "%o"}}) { + name + } + }` + req2 := `query { + User(filter: {email: {_like: "%d@gmail.com"}, name: {_like: "F%"}}) { + name + } + }` + req3 := `query { + User(filter: {email: {_like: "%e%"}, name: {_like: "%n%"}}) { + name + } + }` + req4 := `query { + User(filter: {email: {_like: "fred@gmail.com"}, name: {_like: "Fred"}}) { + name + } + }` + req5 := `query { + User(filter: {email: {_like: "a%@gmail.com"}, name: {_like: "%dd%"}}) { + name + } + }` + req6 := `query { + User(filter: {email: {_like: "a%com%m"}}) { + name + } + }` + req7 := `query { + User(filter: {email: {_like: "s%"}, name: {_like: "s%h%d"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _like filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User @index(fields: ["name", "email"]) { + name: String + email: String + }`, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req1, + Results: []map[string]any{ + {"name": "Addo"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req2, + Results: []map[string]any{ + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req3, + Results: []map[string]any{ + {"name": "Keenan"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req3), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req4, + Results: []map[string]any{ + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req4), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req5, + Results: []map[string]any{ + {"name": "Addo"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req5), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req6, + Results: []map[string]any{}, + }, + testUtils.Request{ + Request: req7, + Results: []map[string]any{}, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithNotLikeFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_nlike: "%h%"}, email: {_nlike: "%d%"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _nlike filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User @index(fields: ["name", "email"]) { + name: String + email: String + }`, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Bruno"}, + {"name": "Islam"}, + {"name": "Keenan"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_IfFirstFieldIsNotInFilter_ShouldNotUseIndex(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test if index is not used when first field is not in filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: `query @explain(type: execute) { + User(filter: {age: {_eq: 32}}) { + name + } + }`, + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(11).WithIndexFetches(0), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From 8fbd099113d11b7f48a65d83387ec25a74731aaa Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 15 Jan 2024 13:41:22 +0100 Subject: [PATCH 20/31] Fix after rebase --- db/fetcher/indexer_iterators.go | 8 ++++---- db/indexed_docs_test.go | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index d181d7e82c..6ccee104ba 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -504,9 +504,9 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { switch fieldConditions[0].op { case opEq: - writableValue := client.NewCBORValue(client.LWW_REGISTER, fieldConditions[0].val) + fieldVal := client.NewFieldValue(client.LWW_REGISTER, fieldConditions[0].val) - keyValueBytes, err := writableValue.Bytes() + keyValueBytes, err := fieldVal.Bytes() if err != nil { return nil, err } @@ -536,8 +536,8 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { } keyFieldArr := make([][]byte, 0, len(inArr)) for _, v := range inArr { - writableValue := client.NewCBORValue(client.LWW_REGISTER, v) - keyFieldBytes, err := writableValue.Bytes() + fieldVal := client.NewFieldValue(client.LWW_REGISTER, v) + keyFieldBytes, err := fieldVal.Bytes() if err != nil { return nil, err } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 17c21c4aae..dd36c63849 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -169,7 +169,7 @@ indexLoop: var fieldBytesVal []byte var fieldValue *client.FieldValue var err error - if len(b.values) == 0 { + if len(b.values) <= i { fieldValue, err = b.doc.GetValue(fieldName) require.NoError(b.f.t, err) } else { @@ -1136,9 +1136,9 @@ func TestCompositeCreate_ShouldIndexExistingDocs(t *testing.T) { f := newIndexTestFixture(t) defer f.db.Close() - doc1 := f.newUserDoc("John", 21) + doc1 := f.newUserDoc("John", 21, f.users) f.saveDocToCollection(doc1, f.users) - doc2 := f.newUserDoc("Islam", 18) + doc2 := f.newUserDoc("Islam", 18, f.users) f.saveDocToCollection(doc2, f.users) f.createUserCollectionIndexOnNameAndAge() @@ -1165,7 +1165,7 @@ func TestComposite_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { }{Age: 44}) require.NoError(f.t, err) - doc, err := client.NewDocFromJSON(docJSON) + doc, err := client.NewDocFromJSON(docJSON, f.users.Schema()) require.NoError(f.t, err) f.saveDocToCollection(doc, f.users) @@ -1187,8 +1187,8 @@ func TestCompositeDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { require.NoError(f.t, err) f.commitTxn() - f.saveDocToCollection(f.newUserDoc("John", 21), users) - f.saveDocToCollection(f.newUserDoc("Islam", 23), users) + f.saveDocToCollection(f.newUserDoc("John", 21, users), users) + f.saveDocToCollection(f.newUserDoc("Islam", 23, users), users) userNameAgeKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersNameFieldName, usersAgeFieldName).Build() userAgeWeightKey := newIndexKeyBuilder(f).Col(usersColName).Fields(usersAgeFieldName, usersWeightFieldName).Build() @@ -1245,7 +1245,7 @@ func TestCompositeUpdate_ShouldDeleteOldValueAndStoreNewOne(t *testing.T) { }, } - doc := f.newUserDoc("John", 21) + doc := f.newUserDoc("John", 21, f.users) f.saveDocToCollection(doc, f.users) for _, tc := range cases { From ad7997ace19b6efb12b369d6dadf5e209bcef824 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 15 Jan 2024 13:59:28 +0100 Subject: [PATCH 21/31] Query compose index on nil values --- ...y_with_composite_index_only_filter_test.go | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/integration/index/query_with_composite_index_only_filter_test.go b/tests/integration/index/query_with_composite_index_only_filter_test.go index e031b3f6ff..96139e54ff 100644 --- a/tests/integration/index/query_with_composite_index_only_filter_test.go +++ b/tests/integration/index/query_with_composite_index_only_filter_test.go @@ -617,3 +617,98 @@ func TestQueryWithCompositeIndex_IfFirstFieldIsNotInFilter_ShouldNotUseIndex(t * testUtils.ExecuteTestCase(t, test) } + +func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFetch(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test index filtering with _eq filter on nil value", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice", + "age": 22 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "age": 32 + }`, + }, + testUtils.Request{ + Request: ` + query { + User(filter: {name: {_eq: null}}) { + age + } + }`, + Results: []map[string]any{ + {"age": 32}, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldFetch(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test index filtering with _eq filter on nil value", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice", + "age": 22 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Bob" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice" + }`, + }, + testUtils.Request{ + Request: ` + query { + User(filter: {name: {_eq: "Alice"}, age: {_eq: null}}) { + name + age + } + }`, + Results: []map[string]any{ + { + "name": "Alice", + "age": nil, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From 770a19108cabe5e08270d6739b9231959c68d13b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 15 Jan 2024 16:50:49 +0100 Subject: [PATCH 22/31] filter on composite index without middle value --- db/fetcher/indexer.go | 2 +- ...y_with_composite_index_only_filter_test.go | 62 ++++++++++++++++++- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/db/fetcher/indexer.go b/db/fetcher/indexer.go index 1b01f884c0..f8cc45225a 100644 --- a/db/fetcher/indexer.go +++ b/db/fetcher/indexer.go @@ -77,7 +77,7 @@ func (f *IndexFetcher) Init( } } - f.docFields = make([]client.FieldDescription, 0, len(fields)-len(f.indexedFields)) + f.docFields = make([]client.FieldDescription, 0, len(fields)) outer: for i := range fields { for j := range f.indexedFields { diff --git a/tests/integration/index/query_with_composite_index_only_filter_test.go b/tests/integration/index/query_with_composite_index_only_filter_test.go index 96139e54ff..4f82f91fa7 100644 --- a/tests/integration/index/query_with_composite_index_only_filter_test.go +++ b/tests/integration/index/query_with_composite_index_only_filter_test.go @@ -620,7 +620,7 @@ func TestQueryWithCompositeIndex_IfFirstFieldIsNotInFilter_ShouldNotUseIndex(t * func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFetch(t *testing.T) { test := testUtils.TestCase{ - Description: "Test index filtering with _eq filter on nil value", + Description: "Test index filtering with _eq filter on nil value on first field", Actions: []any{ testUtils.SchemaUpdate{ Schema: schemaWithNameAgeIndex, @@ -662,7 +662,7 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFetch(t func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldFetch(t *testing.T) { test := testUtils.TestCase{ - Description: "Test index filtering with _eq filter on nil value", + Description: "Test index filtering with _eq filter on nil value on second field", Actions: []any{ testUtils.SchemaUpdate{ Schema: schemaWithNameAgeIndex, @@ -712,3 +712,61 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldFetch(t testUtils.ExecuteTestCase(t, test) } + +func TestQueryWithCompositeIndex_IfMiddleFieldIsNotIfFilter_ShouldIgnoreValue(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test composite index with filter without middle field", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User @index(fields: ["name", "email", "age"]) { + name: String + email: String + age: Int + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice", + "email": "alice@gmail.com", + "age": 22 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alan", + "email": "alan@gmail.com", + "age": 38 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Bob", + "email": "bob@gmail.com", + "age": 51 + }`, + }, + testUtils.Request{ + Request: ` + query { + User(filter: {name: {_like: "%l%"}, age: {_gt: 30}}) { + name + } + }`, + Results: []map[string]any{ + { + "name": "Alan", + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From b220e6b55ad10a2c471da233896212c9c5b93737 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 15 Jan 2024 17:00:18 +0100 Subject: [PATCH 23/31] Fix lint --- db/index.go | 3 +-- planner/filter/copy_field.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/db/index.go b/db/index.go index 9c5419e8b0..44acca7a53 100644 --- a/db/index.go +++ b/db/index.go @@ -131,9 +131,8 @@ func (i *collectionBaseIndex) getDocFieldValue(doc *client.Document) ([][]byte, } result = append(result, valBytes) continue - } else { - return nil, err } + return nil, err } if !i.validateFieldFuncs[iter](fieldVal.Value()) { return nil, NewErrInvalidFieldValue(i.fieldsDescs[iter].Kind, fieldVal) diff --git a/planner/filter/copy_field.go b/planner/filter/copy_field.go index c824e9a9d4..fff974da06 100644 --- a/planner/filter/copy_field.go +++ b/planner/filter/copy_field.go @@ -15,7 +15,7 @@ import ( ) // CopyField copies the given field from the provided filter. -// Multiple fields can be passed to copy related objects with a certain field. +// Multiple fields can be passed to copy related objects with a certain field. // In this case every subsequent field is a sub field of the previous one. Eg. bool.author.name // The result filter preserves the structure of the original filter. func CopyField(filter *mapper.Filter, fields ...mapper.Field) *mapper.Filter { From e30e51bbe79af2c01deba7be0ef7c8d615e8ca04 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 15 Jan 2024 17:02:59 +0100 Subject: [PATCH 24/31] Add copyright header --- client/index_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/index_test.go b/client/index_test.go index cc5c6d79a7..ea7dbb28e9 100644 --- a/client/index_test.go +++ b/client/index_test.go @@ -1,3 +1,13 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package client import ( From 338dd8429c89dc03167afbce8699edc7fefd4b5b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 17 Jan 2024 16:12:17 +0100 Subject: [PATCH 25/31] Add unique composite index --- db/errors.go | 9 + db/fetcher/indexer_iterators.go | 111 ++- db/index.go | 25 +- .../index/create_unique_composite_test.go | 182 ++++ ...y_with_composite_index_only_filter_test.go | 11 +- ...with_unique_composite_index_filter_test.go | 871 ++++++++++++++++++ 6 files changed, 1152 insertions(+), 57 deletions(-) create mode 100644 tests/integration/index/create_unique_composite_test.go create mode 100644 tests/integration/index/query_with_unique_composite_index_filter_test.go diff --git a/db/errors.go b/db/errors.go index d8c9773926..7153ed8017 100644 --- a/db/errors.go +++ b/db/errors.go @@ -87,6 +87,7 @@ const ( errOneOneAlreadyLinked string = "target document is already linked to another document" errIndexDoesNotMatchName string = "the index used does not match the given name" errCanNotIndexNonUniqueField string = "can not index a doc's field that violates unique index" + errCanNotIndexNonUniqueCombination string = "can not index a doc's fields that violate unique index" errInvalidViewQuery string = "the query provided is not valid as a View" ) @@ -575,6 +576,14 @@ func NewErrCanNotIndexNonUniqueField(docID, fieldName string, value any) error { ) } +func NewErrCanNotIndexNonUniqueCombination(docID string, fieldValues ...errors.KV) error { + kvPairs := make([]errors.KV, 0, len(fieldValues)+1) + kvPairs = append(kvPairs, errors.NewKV("DocID", docID)) + kvPairs = append(kvPairs, fieldValues...) + + return errors.New(errCanNotIndexNonUniqueCombination, kvPairs...) +} + func NewErrInvalidViewQueryCastFailed(query string) error { return errors.New( errInvalidViewQuery, diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index 6ccee104ba..e8e3fc6bd7 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -89,10 +89,6 @@ type eqPrefixIndexIterator struct { queryResultIterator } -func (i *eqPrefixIndexIterator) SetKeyFieldValue(value []byte) { - i.keyFieldValue = value -} - func (i *eqPrefixIndexIterator) Init(ctx context.Context, store datastore.DSReaderWriter) error { i.indexKey.FieldValues = [][]byte{i.keyFieldValue} resultIter, err := store.Query(ctx, query.Query{ @@ -123,22 +119,17 @@ func (i *eqPrefixIndexIterator) Next() (indexIterResult, error) { } } -type keyFieldIndexIterator interface { - indexIterator - SetKeyFieldValue([]byte) -} - type eqSingleIndexIterator struct { - indexKey core.IndexDataStoreKey - keyFieldValue []byte - execInfo *ExecInfo + indexKey core.IndexDataStoreKey + keyFieldValues [][]byte + execInfo *ExecInfo ctx context.Context store datastore.DSReaderWriter } func (i *eqSingleIndexIterator) SetKeyFieldValue(value []byte) { - i.keyFieldValue = value + i.keyFieldValues = [][]byte{value} } func (i *eqSingleIndexIterator) Init(ctx context.Context, store datastore.DSReaderWriter) error { @@ -151,7 +142,7 @@ func (i *eqSingleIndexIterator) Next() (indexIterResult, error) { if i.store == nil { return indexIterResult{}, nil } - i.indexKey.FieldValues = [][]byte{i.keyFieldValue} + i.indexKey.FieldValues = i.keyFieldValues val, err := i.store.Get(i.ctx, i.indexKey.ToDS()) if err != nil { if errors.Is(err, ds.ErrNotFound) { @@ -169,7 +160,7 @@ func (i *eqSingleIndexIterator) Close() error { } type inIndexIterator struct { - keyFieldIndexIterator + indexIterator keyFieldValues [][]byte nextValIndex int ctx context.Context @@ -179,7 +170,7 @@ type inIndexIterator struct { func (i *inIndexIterator) nextIterator() (bool, error) { if i.nextValIndex > 0 { - err := i.keyFieldIndexIterator.Close() + err := i.indexIterator.Close() if err != nil { return false, err } @@ -189,8 +180,13 @@ func (i *inIndexIterator) nextIterator() (bool, error) { return false, nil } - i.SetKeyFieldValue(i.keyFieldValues[i.nextValIndex]) - err := i.keyFieldIndexIterator.Init(i.ctx, i.store) + switch fieldIter := i.indexIterator.(type) { + case *eqPrefixIndexIterator: + fieldIter.keyFieldValue = i.keyFieldValues[i.nextValIndex] + case *eqSingleIndexIterator: + fieldIter.keyFieldValues[0] = i.keyFieldValues[i.nextValIndex] + } + err := i.indexIterator.Init(i.ctx, i.store) if err != nil { return false, err } @@ -208,7 +204,7 @@ func (i *inIndexIterator) Init(ctx context.Context, store datastore.DSReaderWrit func (i *inIndexIterator) Next() (indexIterResult, error) { for i.hasIterator { - res, err := i.keyFieldIndexIterator.Next() + res, err := i.indexIterator.Next() if err != nil { return indexIterResult{}, err } @@ -493,6 +489,27 @@ func (f *IndexFetcher) determineFieldFilterConditions() []fieldFilterCond { return result } +func isUniqueFetchByFullKey(indexDesc *client.IndexDescription, conditions []fieldFilterCond) bool { + res := indexDesc.Unique && len(conditions) == len(indexDesc.Fields) + for i := 1; i < len(conditions); i++ { + res = res && conditions[i].op == opEq + } + return res +} + +func getFieldsBytes(conditions []fieldFilterCond) ([][]byte, error) { + result := make([][]byte, 0, len(conditions)) + for i := range conditions { + fieldVal := client.NewFieldValue(client.LWW_REGISTER, conditions[i].val) + keyFieldBytes, err := fieldVal.Bytes() + if err != nil { + return nil, err + } + result = append(result, keyFieldBytes) + } + return result, nil +} + func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { fieldConditions := f.determineFieldFilterConditions() indexDataStoreKey := core.IndexDataStoreKey{CollectionID: f.col.ID(), IndexID: f.indexDesc.ID} @@ -504,24 +521,30 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { switch fieldConditions[0].op { case opEq: - fieldVal := client.NewFieldValue(client.LWW_REGISTER, fieldConditions[0].val) - - keyValueBytes, err := fieldVal.Bytes() - if err != nil { - return nil, err - } - - if len(matchers) > 1 { - matchers[0] = &anyMatcher{} - } - - if f.indexDesc.Unique { + if isUniqueFetchByFullKey(&f.indexDesc, fieldConditions) { + keyFieldsBytes, err := getFieldsBytes(fieldConditions) + if err != nil { + return nil, err + } return &eqSingleIndexIterator{ - indexKey: indexDataStoreKey, - keyFieldValue: keyValueBytes, - execInfo: &f.execInfo, + indexKey: indexDataStoreKey, + keyFieldValues: keyFieldsBytes, + execInfo: &f.execInfo, }, nil } else { + fieldVal := client.NewFieldValue(client.LWW_REGISTER, fieldConditions[0].val) + + keyValueBytes, err := fieldVal.Bytes() + if err != nil { + return nil, err + } + + // iterators for _eq filter already iterate over keys with first field value + // matching the filter value, so we can skip the first matcher + if len(matchers) > 1 { + matchers[0] = &anyMatcher{} + } + return &eqPrefixIndexIterator{ indexKey: indexDataStoreKey, keyFieldValue: keyValueBytes, @@ -544,15 +567,23 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { keyFieldArr = append(keyFieldArr, keyFieldBytes) } + // iterators for _in filter already iterate over keys with first field value + // matching the filter value, so we can skip the first matcher if len(matchers) > 1 { matchers[0] = &anyMatcher{} } - var iter keyFieldIndexIterator - if f.indexDesc.Unique { + var iter indexIterator + if isUniqueFetchByFullKey(&f.indexDesc, fieldConditions) { + restFieldsVals, e := getFieldsBytes(fieldConditions[1:]) + if e != nil { + return nil, e + } + restFieldsVals = append([][]byte{{}}, restFieldsVals...) iter = &eqSingleIndexIterator{ - indexKey: indexDataStoreKey, - execInfo: &f.execInfo, + indexKey: indexDataStoreKey, + execInfo: &f.execInfo, + keyFieldValues: restFieldsVals, } } else { iter = &eqPrefixIndexIterator{ @@ -562,8 +593,8 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { } } return &inIndexIterator{ - keyFieldIndexIterator: iter, - keyFieldValues: keyFieldArr, + indexIterator: iter, + keyFieldValues: keyFieldArr, }, nil case opGt, opGe, opLt, opLe, opNe, opNin, opLike, opNlike: return &scanningIndexIterator{ diff --git a/db/index.go b/db/index.go index 44acca7a53..5313cfb6a2 100644 --- a/db/index.go +++ b/db/index.go @@ -302,19 +302,26 @@ func (i *collectionUniqueIndex) Save( func (i *collectionUniqueIndex) newUniqueIndexError( doc *client.Document, ) error { - fieldVal, err := doc.GetValue(i.fieldsDescs[0].Name) + kvs := make([]errors.KV, 0, len(i.fieldsDescs)) var val any - if err != nil { - // If the error is ErrFieldNotExist, we leave `val` as is (e.g. nil) - // otherwise we return the error - if !errors.Is(err, client.ErrFieldNotExist) { - return err + for iter := range i.fieldsDescs { + fieldVal, err := doc.GetValue(i.fieldsDescs[iter].Name) + if err != nil { + // If the error is ErrFieldNotExist, we leave `val` as is (e.g. nil) + // otherwise we return the error + if !errors.Is(err, client.ErrFieldNotExist) { + return err + } + } else { + val = fieldVal.Value() } - } else { - val = fieldVal.Value() + kvs = append(kvs, errors.NewKV(i.fieldsDescs[iter].Name, val)) } - return NewErrCanNotIndexNonUniqueField(doc.ID().String(), i.fieldsDescs[0].Name, val) + if len(kvs) == 1 { + return NewErrCanNotIndexNonUniqueField(doc.ID().String(), i.fieldsDescs[0].Name, val) + } + return NewErrCanNotIndexNonUniqueCombination(doc.ID().String(), kvs...) } func (i *collectionUniqueIndex) Update( diff --git a/tests/integration/index/create_unique_composite_test.go b/tests/integration/index/create_unique_composite_test.go new file mode 100644 index 0000000000..f9c7b6bd31 --- /dev/null +++ b/tests/integration/index/create_unique_composite_test.go @@ -0,0 +1,182 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/db" + "github.com/sourcenetwork/defradb/errors" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestCreateUniqueCompositeIndex_IfFieldValuesAreNotUnique_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + Description: "If combination of fields is not unique, creating of unique index fails", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + email: String + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "John", + "age": 21, + "email": "email@gmail.com" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "John", + "age": 21, + "email": "another@gmail.com" + }`, + }, + testUtils.CreateIndex{ + CollectionID: 0, + FieldsNames: []string{"name", "age"}, + Unique: true, + ExpectedError: db.NewErrCanNotIndexNonUniqueCombination( + "bae-cae3deac-d371-5a1f-93b4-ede69042f79b", + errors.NewKV("name", "John"), errors.NewKV("age", 21), + ).Error(), + }, + testUtils.GetIndexes{ + CollectionID: 0, + ExpectedIndexes: []client.IndexDescription{}, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestUniqueCompositeIndexCreate_UponAddingDocWithExistingFieldValue_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + Description: "adding a new doc with existing field combination for composite index should fail", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "John", + "age": 21, + "email": "email@gmail.com" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "John", + "age": 21, + "email": "another@gmail.com" + }`, + ExpectedError: db.NewErrCanNotIndexNonUniqueCombination( + "bae-13254430-7e9e-52e2-9861-9a7ec7a75c8d", + errors.NewKV("name", "John"), errors.NewKV("age", 21)).Error(), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestUniqueCompositeIndexCreate_IfFieldValuesAreUnique_Succeed(t *testing.T) { + test := testUtils.TestCase{ + Description: "create unique composite index if all docs have unique fields combinations", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + email: String + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "John", + "age": 21, + "email": "some@gmail.com" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "John", + "age": 35, + "email": "another@gmail.com" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Andy", + "age": 35, + "email": "different@gmail.com" + }`, + }, + testUtils.CreateIndex{ + CollectionID: 0, + FieldsNames: []string{"name", "age"}, + IndexName: "name_age_unique_index", + Unique: true, + }, + testUtils.GetIndexes{ + CollectionID: 0, + ExpectedIndexes: []client.IndexDescription{ + { + Name: "name_age_unique_index", + ID: 1, + Unique: true, + Fields: []client.IndexedFieldDescription{ + { + Name: "name", + Direction: client.Ascending, + }, + { + Name: "age", + Direction: client.Ascending, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/index/query_with_composite_index_only_filter_test.go b/tests/integration/index/query_with_composite_index_only_filter_test.go index 4f82f91fa7..6596c94546 100644 --- a/tests/integration/index/query_with_composite_index_only_filter_test.go +++ b/tests/integration/index/query_with_composite_index_only_filter_test.go @@ -625,9 +625,6 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFetch(t testUtils.SchemaUpdate{ Schema: schemaWithNameAgeIndex, }, - testUtils.CreatePredefinedDocs{ - Docs: getUserDocs(), - }, testUtils.CreateDoc{ CollectionID: 0, Doc: ` @@ -647,11 +644,12 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFetch(t Request: ` query { User(filter: {name: {_eq: null}}) { + name age } }`, Results: []map[string]any{ - {"age": 32}, + {"name": nil, "age": 32}, }, }, }, @@ -667,9 +665,6 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldFetch(t testUtils.SchemaUpdate{ Schema: schemaWithNameAgeIndex, }, - testUtils.CreatePredefinedDocs{ - Docs: getUserDocs(), - }, testUtils.CreateDoc{ CollectionID: 0, Doc: ` @@ -713,7 +708,7 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldFetch(t testUtils.ExecuteTestCase(t, test) } -func TestQueryWithCompositeIndex_IfMiddleFieldIsNotIfFilter_ShouldIgnoreValue(t *testing.T) { +func TestQueryWithCompositeIndex_IfMiddleFieldIsNotInFilter_ShouldIgnoreValue(t *testing.T) { test := testUtils.TestCase{ Description: "Test composite index with filter without middle field", Actions: []any{ diff --git a/tests/integration/index/query_with_unique_composite_index_filter_test.go b/tests/integration/index/query_with_unique_composite_index_filter_test.go new file mode 100644 index 0000000000..a50794c925 --- /dev/null +++ b/tests/integration/index/query_with_unique_composite_index_filter_test.go @@ -0,0 +1,871 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package index + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +const schemaWithNameAgeUniqueIndex = ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }` + +const schemaWithAgeNameUniqueIndex = ` + type User @index(unique: true, fields: ["age", "name"]) { + name: String + age: Int + email: String + }` + +func TestQueryWithUniqueCompositeIndex_WithEqualFilter_ShouldFetch(t *testing.T) { + req1 := `query { + User(filter: {name: {_eq: "Islam"}}) { + name + age + } + }` + req2 := `query { + User(filter: {name: {_eq: "Islam"}, age: {_eq: 32}}) { + name + age + } + }` + req3 := `query { + User(filter: {name: {_eq: "Islam"}, age: {_eq: 66}}) { + name + age + } + }` + test := testUtils.TestCase{ + Description: "Test filtering on composite index with _eq filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Islam", + "age": 40 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Islam", + "age": 50 + }`, + }, + testUtils.Request{ + Request: req1, + Results: []map[string]any{ + {"name": "Islam", "age": 32}, + {"name": "Islam", "age": 40}, + {"name": "Islam", "age": 50}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(3), + }, + testUtils.Request{ + Request: req2, + Results: []map[string]any{ + {"name": "Islam", "age": 32}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(1), + }, + testUtils.Request{ + Request: req3, + Results: []map[string]any{}, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithGreaterThanFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Keenan"}, age: {_gt: 44}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _gt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithGreaterThanFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Keenan"}, age: {_gt: 44}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _gt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithGreaterOrEqualFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Keenan"}, age: {_ge: 44},}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _ge filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithGreaterOrEqualFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_ge: 44}, name: {_ne: "Keenan"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _ge filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Chris"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithLessThanFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_lt: 24}, name: {_ne: "Shahzad"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _lt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Bruno"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithLessThanFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_lt: 24}, name: {_ne: "Shahzad"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _lt filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Bruno"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithLessOrEqualFilterOnFirstField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_le: 28}, name: {_ne: "Bruno"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _le filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithAgeNameUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Shahzad"}, + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithLessOrEqualFilterOnSecondField_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_le: 28}, name: {_ne: "Bruno"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _le filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Fred"}, + {"name": "Shahzad"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithNotEqualFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_ne: "Islam"}, age: {_ne: 28}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _ne filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Addo"}, + {"name": "Andy"}, + {"name": "John"}, + {"name": "Bruno"}, + {"name": "Chris"}, + {"name": "Keenan"}, + {"name": "Shahzad"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(8).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithInForFirstAndEqForRest_ShouldFetchEfficiently(t *testing.T) { + req := `query { + User(filter: {age: {_eq: 33}, name: {_in: ["Addo", "Andy", "Fred"]}}) { + name + age + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _in filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Addo", + "age": 33 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Addo", + "age": 88 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Andy", + "age": 33 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Andy", + "age": 70 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Andy", + "age": 51 + }`, + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Addo", "age": 33}, + {"name": "Andy", "age": 33}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(2), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithInFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_in: [20, 28, 33]}, name: {_in: ["Addo", "Andy", "Fred"]}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _in filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Addo", + "age": 10 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Addo", + "age": 88 + }`, + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Andy"}, + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(5), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithNotInFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {age: {_nin: [20, 23, 28, 42]}, name: {_nin: ["John", "Andy", "Chris"]}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _nin filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Islam"}, + {"name": "Keenan"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithLikeFilter_ShouldFetch(t *testing.T) { + req1 := `query { + User(filter: {email: {_like: "a%"}, name: {_like: "%o"}}) { + name + } + }` + req2 := `query { + User(filter: {email: {_like: "%d@gmail.com"}, name: {_like: "F%"}}) { + name + } + }` + req3 := `query { + User(filter: {email: {_like: "%e%"}, name: {_like: "%n%"}}) { + name + } + }` + req4 := `query { + User(filter: {email: {_like: "fred@gmail.com"}, name: {_like: "Fred"}}) { + name + } + }` + req5 := `query { + User(filter: {email: {_like: "a%@gmail.com"}, name: {_like: "%dd%"}}) { + name + } + }` + req6 := `query { + User(filter: {email: {_like: "a%com%m"}}) { + name + } + }` + req7 := `query { + User(filter: {email: {_like: "s%"}, name: {_like: "s%h%d"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _like filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User @index(unique: true, fields: ["name", "email"]) { + name: String + email: String + }`, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req1, + Results: []map[string]any{ + {"name": "Addo"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req2, + Results: []map[string]any{ + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req3, + Results: []map[string]any{ + {"name": "Keenan"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req3), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req4, + Results: []map[string]any{ + {"name": "Fred"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req4), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req5, + Results: []map[string]any{ + {"name": "Addo"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req5), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(10), + }, + testUtils.Request{ + Request: req6, + Results: []map[string]any{}, + }, + testUtils.Request{ + Request: req7, + Results: []map[string]any{}, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithNotLikeFilter_ShouldFetch(t *testing.T) { + req := `query { + User(filter: {name: {_nlike: "%h%"}, email: {_nlike: "%d%"}}) { + name + } + }` + test := testUtils.TestCase{ + Description: "Test index filtering with _nlike filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User @index(unique: true, fields: ["name", "email"]) { + name: String + email: String + }`, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"name": "Roy"}, + {"name": "Bruno"}, + {"name": "Islam"}, + {"name": "Keenan"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(10), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_IfFirstFieldIsNotInFilter_ShouldNotUseIndex(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test if index is not used when first field is not in filter", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: `query @explain(type: execute) { + User(filter: {age: {_eq: 32}}) { + name + } + }`, + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(11).WithIndexFetches(0), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFetch(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test index filtering with _eq filter on nil value on first field", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice", + "age": 22 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "age": 32 + }`, + }, + testUtils.Request{ + Request: ` + query { + User(filter: {name: {_eq: null}}) { + name + age + } + }`, + Results: []map[string]any{ + {"name": nil, "age": 32}, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldFetch(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test index filtering with _eq filter on nil value on second field", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: schemaWithNameAgeUniqueIndex, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice", + "age": 22 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Bob" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice" + }`, + }, + testUtils.Request{ + Request: ` + query { + User(filter: {name: {_eq: "Alice"}, age: {_eq: null}}) { + name + age + } + }`, + Results: []map[string]any{ + { + "name": "Alice", + "age": nil, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithUniqueCompositeIndex_IfMiddleFieldIsNotInFilter_ShouldIgnoreValue(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test composite index with filter without middle field", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User @index(unique: true, fields: ["name", "email", "age"]) { + name: String + email: String + age: Int + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alice", + "email": "alice@gmail.com", + "age": 22 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Alan", + "email": "alan@gmail.com", + "age": 38 + }`, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: ` + { + "name": "Bob", + "email": "bob@gmail.com", + "age": 51 + }`, + }, + testUtils.Request{ + Request: ` + query { + User(filter: {name: {_like: "%l%"}, age: {_gt: 30}}) { + name + } + }`, + Results: []map[string]any{ + { + "name": "Alan", + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From 20595138f6c617c7a0e7678c7dbe74d7ab2dedc9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 Jan 2024 15:45:12 +0100 Subject: [PATCH 26/31] Rename method, add comment --- client/index.go | 4 +++- client/index_test.go | 2 +- planner/planner.go | 2 +- planner/select.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/client/index.go b/client/index.go index 7fb59658b5..d0726b1625 100644 --- a/client/index.go +++ b/client/index.go @@ -59,7 +59,9 @@ func (d CollectionDescription) CollectIndexedFields(schema *SchemaDescription) [ return fields } -func (d CollectionDescription) CollectIndexesOnField(fieldName string) []IndexDescription { +// GetIndexesOnField returns all indexes that are indexing the given field. +// If the field is not the first field of a composite index, the index is not returned. +func (d CollectionDescription) GetIndexesOnField(fieldName string) []IndexDescription { result := []IndexDescription{} for _, index := range d.Indexes { if index.Fields[0].Name == fieldName { diff --git a/client/index_test.go b/client/index_test.go index ea7dbb28e9..feb8ccdd69 100644 --- a/client/index_test.go +++ b/client/index_test.go @@ -122,7 +122,7 @@ func TestCollectIndexesOnField(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual := tt.desc.CollectIndexesOnField(tt.field) + actual := tt.desc.GetIndexesOnField(tt.field) assert.Equal(t, tt.expected, actual) }) } diff --git a/planner/planner.go b/planner/planner.go index d92106d877..3ef8ff28e3 100644 --- a/planner/planner.go +++ b/planner/planner.go @@ -343,7 +343,7 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan slct := node.subType.(*selectTopNode).selectNode desc := slct.collection.Description() for subFieldName, subFieldInd := range filteredSubFields { - indexes := desc.CollectIndexesOnField(subFieldName) + indexes := desc.GetIndexesOnField(subFieldName) if len(indexes) > 0 { subInd := node.documentMapping.FirstIndexOfName(node.subTypeName) relatedField := mapper.Field{Name: node.subTypeName, Index: subInd} diff --git a/planner/select.go b/planner/select.go index fafc6f2788..9163816ccd 100644 --- a/planner/select.go +++ b/planner/select.go @@ -304,7 +304,7 @@ func findFilteredByIndexedField(scanNode *scanNode) immutable.Option[client.Inde if _, isFiltered := scanNode.filter.ExternalConditions[field.Name]; !isFiltered { continue } - indexes := colDesc.CollectIndexesOnField(field.Name) + indexes := colDesc.GetIndexesOnField(field.Name) if len(indexes) > 0 { return immutable.Some(indexes[0]) } From c053e7d4e1a9da239edbce3eee10503ad51235bb Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 Jan 2024 16:14:04 +0100 Subject: [PATCH 27/31] Merge 2 errors into 1 --- db/errors.go | 17 +++---------- db/index.go | 7 ++---- .../index/create_unique_composite_test.go | 4 +-- tests/integration/index/create_unique_test.go | 25 +++++++++++-------- 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/db/errors.go b/db/errors.go index 7153ed8017..c998952232 100644 --- a/db/errors.go +++ b/db/errors.go @@ -86,8 +86,7 @@ const ( errExpectedJSONArray string = "expected JSON array" errOneOneAlreadyLinked string = "target document is already linked to another document" errIndexDoesNotMatchName string = "the index used does not match the given name" - errCanNotIndexNonUniqueField string = "can not index a doc's field that violates unique index" - errCanNotIndexNonUniqueCombination string = "can not index a doc's fields that violate unique index" + errCanNotIndexNonUniqueFields string = "can not index a doc's field(s) that violates unique index" errInvalidViewQuery string = "the query provided is not valid as a View" ) @@ -109,6 +108,7 @@ var ( ErrExpectedJSONObject = errors.New(errExpectedJSONObject) ErrExpectedJSONArray = errors.New(errExpectedJSONArray) ErrInvalidViewQuery = errors.New(errInvalidViewQuery) + ErrCanNotIndexNonUniqueFields = errors.New(errCanNotIndexNonUniqueFields) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document @@ -567,21 +567,12 @@ func NewErrIndexDoesNotMatchName(index, name string) error { ) } -func NewErrCanNotIndexNonUniqueField(docID, fieldName string, value any) error { - return errors.New( - errCanNotIndexNonUniqueField, - errors.NewKV("DocID", docID), - errors.NewKV("Field name", fieldName), - errors.NewKV("Field value", value), - ) -} - -func NewErrCanNotIndexNonUniqueCombination(docID string, fieldValues ...errors.KV) error { +func NewErrCanNotIndexNonUniqueFields(docID string, fieldValues ...errors.KV) error { kvPairs := make([]errors.KV, 0, len(fieldValues)+1) kvPairs = append(kvPairs, errors.NewKV("DocID", docID)) kvPairs = append(kvPairs, fieldValues...) - return errors.New(errCanNotIndexNonUniqueCombination, kvPairs...) + return errors.New(errCanNotIndexNonUniqueFields, kvPairs...) } func NewErrInvalidViewQueryCastFailed(query string) error { diff --git a/db/index.go b/db/index.go index 5313cfb6a2..67712307ae 100644 --- a/db/index.go +++ b/db/index.go @@ -303,9 +303,9 @@ func (i *collectionUniqueIndex) newUniqueIndexError( doc *client.Document, ) error { kvs := make([]errors.KV, 0, len(i.fieldsDescs)) - var val any for iter := range i.fieldsDescs { fieldVal, err := doc.GetValue(i.fieldsDescs[iter].Name) + var val any if err != nil { // If the error is ErrFieldNotExist, we leave `val` as is (e.g. nil) // otherwise we return the error @@ -318,10 +318,7 @@ func (i *collectionUniqueIndex) newUniqueIndexError( kvs = append(kvs, errors.NewKV(i.fieldsDescs[iter].Name, val)) } - if len(kvs) == 1 { - return NewErrCanNotIndexNonUniqueField(doc.ID().String(), i.fieldsDescs[0].Name, val) - } - return NewErrCanNotIndexNonUniqueCombination(doc.ID().String(), kvs...) + return NewErrCanNotIndexNonUniqueFields(doc.ID().String(), kvs...) } func (i *collectionUniqueIndex) Update( diff --git a/tests/integration/index/create_unique_composite_test.go b/tests/integration/index/create_unique_composite_test.go index f9c7b6bd31..3d146eb591 100644 --- a/tests/integration/index/create_unique_composite_test.go +++ b/tests/integration/index/create_unique_composite_test.go @@ -54,7 +54,7 @@ func TestCreateUniqueCompositeIndex_IfFieldValuesAreNotUnique_ReturnError(t *tes CollectionID: 0, FieldsNames: []string{"name", "age"}, Unique: true, - ExpectedError: db.NewErrCanNotIndexNonUniqueCombination( + ExpectedError: db.NewErrCanNotIndexNonUniqueFields( "bae-cae3deac-d371-5a1f-93b4-ede69042f79b", errors.NewKV("name", "John"), errors.NewKV("age", 21), ).Error(), @@ -99,7 +99,7 @@ func TestUniqueCompositeIndexCreate_UponAddingDocWithExistingFieldValue_ReturnEr "age": 21, "email": "another@gmail.com" }`, - ExpectedError: db.NewErrCanNotIndexNonUniqueCombination( + ExpectedError: db.NewErrCanNotIndexNonUniqueFields( "bae-13254430-7e9e-52e2-9861-9a7ec7a75c8d", errors.NewKV("name", "John"), errors.NewKV("age", 21)).Error(), }, diff --git a/tests/integration/index/create_unique_test.go b/tests/integration/index/create_unique_test.go index fac2330a28..303e91589a 100644 --- a/tests/integration/index/create_unique_test.go +++ b/tests/integration/index/create_unique_test.go @@ -15,6 +15,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/db" + "github.com/sourcenetwork/defradb/errors" testUtils "github.com/sourcenetwork/defradb/tests/integration" ) @@ -57,10 +58,11 @@ func TestCreateUniqueIndex_IfFieldValuesAreNotUnique_ReturnError(t *testing.T) { }`, }, testUtils.CreateIndex{ - CollectionID: 0, - FieldName: "age", - Unique: true, - ExpectedError: db.NewErrCanNotIndexNonUniqueField(johnDocID, "age", 21).Error(), + CollectionID: 0, + FieldName: "age", + Unique: true, + ExpectedError: db.NewErrCanNotIndexNonUniqueFields( + johnDocID, errors.NewKV("age", 21)).Error(), }, testUtils.GetIndexes{ CollectionID: 0, @@ -99,7 +101,8 @@ func TestUniqueIndexCreate_UponAddingDocWithExistingFieldValue_ReturnError(t *te "name": "John", "age": 21 }`, - ExpectedError: db.NewErrCanNotIndexNonUniqueField(johnDocID, "age", 21).Error(), + ExpectedError: db.NewErrCanNotIndexNonUniqueFields( + johnDocID, errors.NewKV("age", 21)).Error(), }, testUtils.Request{ Request: `query { @@ -222,10 +225,11 @@ func TestUniqueIndexCreate_IfNilFieldsArePresent_ReturnError(t *testing.T) { }`, }, testUtils.CreateIndex{ - CollectionID: 0, - FieldName: "age", - Unique: true, - ExpectedError: db.NewErrCanNotIndexNonUniqueField("bae-caba9876-89aa-5bcf-bc1c-387a52499b27", "age", nil).Error(), + CollectionID: 0, + FieldName: "age", + Unique: true, + ExpectedError: db.NewErrCanNotIndexNonUniqueFields( + "bae-caba9876-89aa-5bcf-bc1c-387a52499b27", errors.NewKV("age", nil)).Error(), }, }, } @@ -291,7 +295,8 @@ func TestUniqueIndexCreate_UponAddingDocWithExistingNilValue_ReturnError(t *test { "name": "Andy" }`, - ExpectedError: db.NewErrCanNotIndexNonUniqueField("bae-2159860f-3cd1-59de-9440-71331e77cbb8", "age", nil).Error(), + ExpectedError: db.NewErrCanNotIndexNonUniqueFields( + "bae-2159860f-3cd1-59de-9440-71331e77cbb8", errors.NewKV("age", nil)).Error(), }, }, } From 2022a936b53e719305fb89b775c1e040d539604f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 Jan 2024 16:16:40 +0100 Subject: [PATCH 28/31] Add a comment --- db/fetcher/indexer_iterators.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index e8e3fc6bd7..82bf928b74 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -39,7 +39,12 @@ const ( opNin = "_nin" opLike = "_like" opNlike = "_nlike" - opAny = "_any" + // it's just there for composite indexes. We construct a slice of value matchers with + // every matcher being responsible for a corresponding field in the index to match. + // For some fields there might not be any criteria to match. For examples if you have + // composite index of /name/age/email/ and in the filter you specify only "name" and "email". + // Then the "_any" matcher will be used for "age". + opAny = "_any" ) // indexIterator is an iterator over index keys. From aaa0bc62032352db51f7c567c6a310e1645d120e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 Jan 2024 19:50:51 +0100 Subject: [PATCH 29/31] PR fixup --- db/errors.go | 11 ----------- db/fetcher/errors.go | 4 ++++ db/fetcher/indexer_iterators.go | 17 +++++++++++------ db/index.go | 2 +- db/index_test.go | 2 +- db/indexed_docs_test.go | 2 +- planner/select.go | 27 ++++++++++++++------------- 7 files changed, 32 insertions(+), 33 deletions(-) diff --git a/db/errors.go b/db/errors.go index c998952232..9b1f414fd6 100644 --- a/db/errors.go +++ b/db/errors.go @@ -69,7 +69,6 @@ const ( errInvalidFieldValue string = "invalid field value" errUnsupportedIndexFieldType string = "unsupported index field type" errIndexDescriptionHasNoFields string = "index description has no fields" - errIndexDescHasNonExistingField string = "index description has non existing field" errFieldOrAliasToFieldNotExist string = "The given field or alias to field does not exist" errCreateFile string = "failed to create file" errRemoveFile string = "failed to remove file" @@ -469,16 +468,6 @@ func NewErrIndexDescHasNoFields(desc client.IndexDescription) error { ) } -// NewErrIndexDescHasNonExistingField returns a new error indicating that the given index -// description points to a field that does not exist. -func NewErrIndexDescHasNonExistingField(desc client.IndexDescription, fieldName string) error { - return errors.New( - errIndexDescHasNonExistingField, - errors.NewKV("Description", desc), - errors.NewKV("Field name", fieldName), - ) -} - // NewErrCreateFile returns a new error indicating there was a failure in creating a file. func NewErrCreateFile(inner error, filepath string) error { return errors.Wrap(errCreateFile, inner, errors.NewKV("Filepath", filepath)) diff --git a/db/fetcher/errors.go b/db/fetcher/errors.go index 84d947c46f..6e4f3a6abb 100644 --- a/db/fetcher/errors.go +++ b/db/fetcher/errors.go @@ -26,6 +26,8 @@ const ( errVFetcherFailedToGetDagLink string = "(version fetcher) failed to get node link from DAG" errFailedToGetDagNode string = "failed to get DAG Node" errMissingMapper string = "missing document mapper" + errInvalidInOperatorValue string = "invalid _in/_nin value" + errInvalidIndexFilterCondition string = "invalid index filter condition" ) var ( @@ -41,6 +43,8 @@ var ( ErrFailedToGetDagNode = errors.New(errFailedToGetDagNode) ErrMissingMapper = errors.New(errMissingMapper) ErrSingleSpanOnly = errors.New("spans must contain only a single entry") + ErrInvalidInOperatorValue = errors.New(errInvalidInOperatorValue) + ErrInvalidIndexFilterCondition = errors.New(errInvalidIndexFilterCondition) ) // NewErrFieldIdNotFound returns an error indicating that the given FieldId was not found. diff --git a/db/fetcher/indexer_iterators.go b/db/fetcher/indexer_iterators.go index 82bf928b74..76786f5050 100644 --- a/db/fetcher/indexer_iterators.go +++ b/db/fetcher/indexer_iterators.go @@ -425,7 +425,7 @@ func createValueMatcher(op string, filterVal any) (valueMatcher, error) { case opIn, opNin: inArr, ok := filterVal.([]any) if !ok { - return nil, errors.New("invalid _in/_nin value") + return nil, ErrInvalidInOperatorValue } valArr := make([][]byte, 0, len(inArr)) for _, v := range inArr { @@ -443,7 +443,7 @@ func createValueMatcher(op string, filterVal any) (valueMatcher, error) { return &anyMatcher{}, nil } - return nil, errors.New("invalid index filter condition") + return nil, ErrInvalidIndexFilterCondition } func createValueMatchers(conditions []fieldFilterCond) ([]valueMatcher, error) { @@ -468,12 +468,10 @@ func (f *IndexFetcher) determineFieldFilterConditions() []fieldFilterCond { for i := range f.indexedFields { fieldInd := f.mapping.FirstIndexOfName(f.indexedFields[i].Name) found := false + // iterate through conditions and find the one that matches the current field for filterKey, indexFilterCond := range f.indexFilter.Conditions { propKey, ok := filterKey.(*mapper.PropertyIndex) - if !ok { - continue - } - if fieldInd != propKey.Index { + if !ok || fieldInd != propKey.Index { continue } @@ -494,7 +492,14 @@ func (f *IndexFetcher) determineFieldFilterConditions() []fieldFilterCond { return result } +// isUniqueFetchByFullKey checks if the only index key can be fetched by the full index key. +// +// This method ignores the first condition because it's expected to be called only +// when the first field is used as a prefix in the index key. So we only check if the +// rest of the conditions are _eq. func isUniqueFetchByFullKey(indexDesc *client.IndexDescription, conditions []fieldFilterCond) bool { + // we need to check length of conditions because full key fetch is only possible + // if all fields are specified in the filter res := indexDesc.Unique && len(conditions) == len(indexDesc.Fields) for i := 1; i < len(conditions); i++ { res = res && conditions[i].op == opEq diff --git a/db/index.go b/db/index.go index 67712307ae..d76573de65 100644 --- a/db/index.go +++ b/db/index.go @@ -96,7 +96,7 @@ func NewCollectionIndex( for _, fieldDesc := range desc.Fields { field, foundField := collection.Schema().GetField(fieldDesc.Name) if !foundField { - return nil, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name) + return nil, client.NewErrFieldNotExist(desc.Fields[0].Name) } base.fieldsDescs = append(base.fieldsDescs, field) validateFunc, err := getFieldValidateFunc(field.Kind) diff --git a/db/index_test.go b/db/index_test.go index feb60dbb8e..30d3258674 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -1287,5 +1287,5 @@ func TestNewCollectionIndex_IfDescriptionHasNonExistingField_ReturnError(t *test desc := getUsersIndexDescOnName() desc.Fields[0].Name = "non_existing_field" _, err := NewCollectionIndex(f.users, desc) - require.ErrorIs(t, err, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name)) + require.ErrorIs(t, err, client.NewErrFieldNotExist(desc.Fields[0].Name)) } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index dd36c63849..9b88bb05f5 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -95,7 +95,7 @@ func (b *indexKeyBuilder) Col(colName string) *indexKeyBuilder { return b } -// Fields sets the fields' names for the index key. +// Fields sets the fields names for the index key. // If the field name is not set, the index key will contain only collection id. // When building a key it will it will find the field id to use in the key. func (b *indexKeyBuilder) Fields(fieldsNames ...string) *indexKeyBuilder { diff --git a/planner/select.go b/planner/select.go index 9163816ccd..ce7ff19030 100644 --- a/planner/select.go +++ b/planner/select.go @@ -290,27 +290,28 @@ func (n *selectNode) initSource() ([]aggregateNode, error) { } if isScanNode { - origScan.initFetcher(n.selectReq.Cid, findFilteredByIndexedField(origScan)) + origScan.initFetcher(n.selectReq.Cid, findIndexByFilteringField(origScan)) } return aggregates, nil } -func findFilteredByIndexedField(scanNode *scanNode) immutable.Option[client.IndexDescription] { - if scanNode.filter != nil { - colDesc := scanNode.col.Description() +func findIndexByFilteringField(scanNode *scanNode) immutable.Option[client.IndexDescription] { + if scanNode.filter == nil { + return immutable.None[client.IndexDescription]() + } + colDesc := scanNode.col.Description() - for _, field := range scanNode.col.Schema().Fields { - if _, isFiltered := scanNode.filter.ExternalConditions[field.Name]; !isFiltered { - continue - } - indexes := colDesc.GetIndexesOnField(field.Name) - if len(indexes) > 0 { - return immutable.Some(indexes[0]) - } + for _, field := range scanNode.col.Schema().Fields { + if _, isFiltered := scanNode.filter.ExternalConditions[field.Name]; !isFiltered { + continue + } + indexes := colDesc.GetIndexesOnField(field.Name) + if len(indexes) > 0 { + // we return the first found index. We will optimize it later. + return immutable.Some(indexes[0]) } } - return immutable.None[client.IndexDescription]() } From 28ec9add24a83d4a6c773bc1765a9383ed08b6f6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 19 Jan 2024 22:20:28 +0100 Subject: [PATCH 30/31] Write schema string directly in test action --- ...y_with_composite_index_only_filter_test.go | 119 +++++++++++++---- ...with_unique_composite_index_filter_test.go | 126 +++++++++++++----- 2 files changed, 186 insertions(+), 59 deletions(-) diff --git a/tests/integration/index/query_with_composite_index_only_filter_test.go b/tests/integration/index/query_with_composite_index_only_filter_test.go index 6596c94546..ce09ec1f89 100644 --- a/tests/integration/index/query_with_composite_index_only_filter_test.go +++ b/tests/integration/index/query_with_composite_index_only_filter_test.go @@ -16,20 +16,6 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -const schemaWithNameAgeIndex = ` - type User @index(fields: ["name", "age"]) { - name: String - age: Int - email: String - }` - -const schemaWithAgeNameIndex = ` - type User @index(fields: ["age", "name"]) { - name: String - age: Int - email: String - }` - func TestQueryWithCompositeIndex_WithEqualFilter_ShouldFetch(t *testing.T) { req1 := `query { User(filter: {name: {_eq: "Islam"}}) { @@ -53,7 +39,12 @@ func TestQueryWithCompositeIndex_WithEqualFilter_ShouldFetch(t *testing.T) { Description: "Test filtering on composite index with _eq filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -98,7 +89,12 @@ func TestQueryWithCompositeIndex_WithGreaterThanFilterOnFirstField_ShouldFetch(t Description: "Test index filtering with _gt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameIndex, + Schema: ` + type User @index(fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -129,7 +125,12 @@ func TestQueryWithCompositeIndex_WithGreaterThanFilterOnSecondField_ShouldFetch( Description: "Test index filtering with _gt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -160,7 +161,12 @@ func TestQueryWithCompositeIndex_WithGreaterOrEqualFilterOnFirstField_ShouldFetc Description: "Test index filtering with _ge filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameIndex, + Schema: ` + type User @index(fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -192,7 +198,12 @@ func TestQueryWithCompositeIndex_WithGreaterOrEqualFilterOnSecondField_ShouldFet Description: "Test index filtering with _ge filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -224,7 +235,12 @@ func TestQueryWithCompositeIndex_WithLessThanFilterOnFirstField_ShouldFetch(t *t Description: "Test index filtering with _lt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameIndex, + Schema: ` + type User @index(fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -255,7 +271,12 @@ func TestQueryWithCompositeIndex_WithLessThanFilterOnSecondField_ShouldFetch(t * Description: "Test index filtering with _lt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -286,7 +307,12 @@ func TestQueryWithCompositeIndex_WithLessOrEqualFilterOnFirstField_ShouldFetch(t Description: "Test index filtering with _le filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameIndex, + Schema: ` + type User @index(fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -318,7 +344,12 @@ func TestQueryWithCompositeIndex_WithLessOrEqualFilterOnSecondField_ShouldFetch( Description: "Test index filtering with _le filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -350,7 +381,12 @@ func TestQueryWithCompositeIndex_WithNotEqualFilter_ShouldFetch(t *testing.T) { Description: "Test index filtering with _ne filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -388,7 +424,12 @@ func TestQueryWithCompositeIndex_WithInFilter_ShouldFetch(t *testing.T) { Description: "Test index filtering with _in filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -420,7 +461,12 @@ func TestQueryWithCompositeIndex_WithNotInFilter_ShouldFetch(t *testing.T) { Description: "Test index filtering with _nin filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -599,7 +645,12 @@ func TestQueryWithCompositeIndex_IfFirstFieldIsNotInFilter_ShouldNotUseIndex(t * Description: "Test if index is not used when first field is not in filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -623,7 +674,12 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFetch(t Description: "Test index filtering with _eq filter on nil value on first field", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreateDoc{ CollectionID: 0, @@ -663,7 +719,12 @@ func TestQueryWithCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldFetch(t Description: "Test index filtering with _eq filter on nil value on second field", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeIndex, + Schema: ` + type User @index(fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreateDoc{ CollectionID: 0, diff --git a/tests/integration/index/query_with_unique_composite_index_filter_test.go b/tests/integration/index/query_with_unique_composite_index_filter_test.go index a50794c925..17e1ac76ea 100644 --- a/tests/integration/index/query_with_unique_composite_index_filter_test.go +++ b/tests/integration/index/query_with_unique_composite_index_filter_test.go @@ -16,20 +16,6 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -const schemaWithNameAgeUniqueIndex = ` - type User @index(unique: true, fields: ["name", "age"]) { - name: String - age: Int - email: String - }` - -const schemaWithAgeNameUniqueIndex = ` - type User @index(unique: true, fields: ["age", "name"]) { - name: String - age: Int - email: String - }` - func TestQueryWithUniqueCompositeIndex_WithEqualFilter_ShouldFetch(t *testing.T) { req1 := `query { User(filter: {name: {_eq: "Islam"}}) { @@ -53,7 +39,12 @@ func TestQueryWithUniqueCompositeIndex_WithEqualFilter_ShouldFetch(t *testing.T) Description: "Test filtering on composite index with _eq filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -116,7 +107,12 @@ func TestQueryWithUniqueCompositeIndex_WithGreaterThanFilterOnFirstField_ShouldF Description: "Test index filtering with _gt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -147,7 +143,12 @@ func TestQueryWithUniqueCompositeIndex_WithGreaterThanFilterOnSecondField_Should Description: "Test index filtering with _gt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -178,7 +179,12 @@ func TestQueryWithUniqueCompositeIndex_WithGreaterOrEqualFilterOnFirstField_Shou Description: "Test index filtering with _ge filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -210,7 +216,12 @@ func TestQueryWithUniqueCompositeIndex_WithGreaterOrEqualFilterOnSecondField_Sho Description: "Test index filtering with _ge filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -242,7 +253,12 @@ func TestQueryWithUniqueCompositeIndex_WithLessThanFilterOnFirstField_ShouldFetc Description: "Test index filtering with _lt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -273,7 +289,12 @@ func TestQueryWithUniqueCompositeIndex_WithLessThanFilterOnSecondField_ShouldFet Description: "Test index filtering with _lt filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -304,7 +325,12 @@ func TestQueryWithUniqueCompositeIndex_WithLessOrEqualFilterOnFirstField_ShouldF Description: "Test index filtering with _le filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithAgeNameUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["age", "name"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -336,7 +362,12 @@ func TestQueryWithUniqueCompositeIndex_WithLessOrEqualFilterOnSecondField_Should Description: "Test index filtering with _le filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -368,7 +399,12 @@ func TestQueryWithUniqueCompositeIndex_WithNotEqualFilter_ShouldFetch(t *testing Description: "Test index filtering with _ne filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -407,7 +443,12 @@ func TestQueryWithUniqueCompositeIndex_WithInForFirstAndEqForRest_ShouldFetchEff Description: "Test index filtering with _in filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreateDoc{ CollectionID: 0, @@ -476,7 +517,12 @@ func TestQueryWithUniqueCompositeIndex_WithInFilter_ShouldFetch(t *testing.T) { Description: "Test index filtering with _in filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -524,7 +570,12 @@ func TestQueryWithUniqueCompositeIndex_WithNotInFilter_ShouldFetch(t *testing.T) Description: "Test index filtering with _nin filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -703,7 +754,12 @@ func TestQueryWithUniqueCompositeIndex_IfFirstFieldIsNotInFilter_ShouldNotUseInd Description: "Test if index is not used when first field is not in filter", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreatePredefinedDocs{ Docs: getUserDocs(), @@ -727,7 +783,12 @@ func TestQueryWithUniqueCompositeIndex_WithEqualFilterOnNilValueOnFirst_ShouldFe Description: "Test index filtering with _eq filter on nil value on first field", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreateDoc{ CollectionID: 0, @@ -767,7 +828,12 @@ func TestQueryWithUniqueCompositeIndex_WithEqualFilterOnNilValueOnSecond_ShouldF Description: "Test index filtering with _eq filter on nil value on second field", Actions: []any{ testUtils.SchemaUpdate{ - Schema: schemaWithNameAgeUniqueIndex, + Schema: ` + type User @index(unique: true, fields: ["name", "age"]) { + name: String + age: Int + email: String + }`, }, testUtils.CreateDoc{ CollectionID: 0, From fbb69d23c354df09d300a87c512874b286b3a8cc Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 22 Jan 2024 12:13:50 +0100 Subject: [PATCH 31/31] Fix after rebase --- db/index_test.go | 2 +- db/indexed_docs_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/index_test.go b/db/index_test.go index 30d3258674..ba7d62e8de 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -207,7 +207,7 @@ func addFieldToIndex(indexDesc client.IndexDescription, fieldName string) client func (f *indexTestFixture) createUserCollectionIndexOnNameAndAge() client.IndexDescription { indexDesc := addFieldToIndex(getUsersIndexDescOnName(), usersAgeFieldName) - newDesc, err := f.createCollectionIndexFor(f.users.Name(), indexDesc) + newDesc, err := f.createCollectionIndexFor(f.users.Name().Value(), indexDesc) require.NoError(f.t, err) return newDesc } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 9b88bb05f5..68d89bcde8 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -1181,9 +1181,9 @@ func TestComposite_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { func TestCompositeDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f := newIndexTestFixtureBare(t) users := f.addUsersCollection() - _, err := f.createCollectionIndexFor(users.Name(), addFieldToIndex(getUsersIndexDescOnName(), usersAgeFieldName)) + _, err := f.createCollectionIndexFor(users.Name().Value(), addFieldToIndex(getUsersIndexDescOnName(), usersAgeFieldName)) require.NoError(f.t, err) - _, err = f.createCollectionIndexFor(users.Name(), addFieldToIndex(getUsersIndexDescOnAge(), usersWeightFieldName)) + _, err = f.createCollectionIndexFor(users.Name().Value(), addFieldToIndex(getUsersIndexDescOnAge(), usersWeightFieldName)) require.NoError(f.t, err) f.commitTxn()