From 7afd32eb04baaefc5cedab028316c7802bb5ffb6 Mon Sep 17 00:00:00 2001 From: AndrewSisley Date: Mon, 16 Oct 2023 16:55:28 -0400 Subject: [PATCH] refactor: Deprecate CollectionDescription.Schema (#1939) ## Relevant issue(s) Resolves #1955 ## Description Deprecate CollectionDescription.Schema. Schema is not a sub-property of collection. Removes as many references to CollectionDescription.Schema as possible without making any breaking changes. Breaking changes will be made in a later PR. An exception has been made to the http API, which does have a breaking change in this PR. --- cli/collection_describe.go | 6 +- client/collection.go | 19 +- client/descriptions.go | 49 ++- core/parser.go | 4 +- db/base/collection_keys.go | 20 +- db/collection.go | 239 ++++++----- db/collection_delete.go | 4 +- db/collection_get.go | 5 +- db/collection_index.go | 25 +- db/collection_test.go | 309 --------------- db/collection_update.go | 10 +- db/db_test.go | 372 ------------------ db/errors.go | 2 +- db/fetcher/fetcher.go | 19 +- db/fetcher/indexer.go | 10 +- db/fetcher/mocks/fetcher.go | 12 +- db/fetcher/versioned.go | 13 +- db/fetcher_test.go | 156 -------- db/index.go | 6 +- db/index_test.go | 278 ++++++------- db/indexed_docs_test.go | 65 ++- db/p2p_collection_test.go | 34 +- db/schema.go | 68 ++-- http/client.go | 20 +- http/client_collection.go | 46 ++- http/handler_store.go | 10 +- lens/fetcher.go | 20 +- net/process.go | 2 +- planner/commit.go | 2 +- planner/datasource.go | 16 +- planner/mapper/descriptions.go | 66 ---- planner/mapper/mapper.go | 160 +++++--- planner/planner.go | 13 +- planner/scan.go | 28 +- planner/select.go | 19 +- planner/type_join.go | 17 +- request/graphql/parser.go | 4 +- request/graphql/schema/collection.go | 36 +- request/graphql/schema/descriptions_test.go | 96 +++-- request/graphql/schema/generate.go | 16 +- request/graphql/schema/index_test.go | 4 +- tests/clients/cli/wrapper.go | 20 +- tests/clients/cli/wrapper_collection.go | 54 +-- .../schema/updates/remove/simple_test.go | 2 +- 44 files changed, 761 insertions(+), 1615 deletions(-) delete mode 100644 planner/mapper/descriptions.go diff --git a/cli/collection_describe.go b/cli/collection_describe.go index 1d6ee55821..a21c4d0c10 100644 --- a/cli/collection_describe.go +++ b/cli/collection_describe.go @@ -39,16 +39,16 @@ Example: view collection by version id col, ok := tryGetCollectionContext(cmd) if ok { - return writeJSON(cmd, col.Description()) + return writeJSON(cmd, col.Definition()) } // if no collection specified list all collections cols, err := store.GetAllCollections(cmd.Context()) if err != nil { return err } - colDesc := make([]client.CollectionDescription, len(cols)) + colDesc := make([]client.CollectionDefinition, len(cols)) for i, col := range cols { - colDesc[i] = col.Description() + colDesc[i] = col.Definition() } return writeJSON(cmd, colDesc) }, diff --git a/client/collection.go b/client/collection.go index 9c91dccb7c..f52eeec0c2 100644 --- a/client/collection.go +++ b/client/collection.go @@ -16,6 +16,14 @@ import ( "github.com/sourcenetwork/defradb/datastore" ) +// CollectionDefinition contains the metadata defining what a Collection is. +type CollectionDefinition struct { + // Description returns the CollectionDescription of this Collection. + Description CollectionDescription `json:"description"` + // Schema returns the SchemaDescription used to define this Collection. + Schema SchemaDescription `json:"schema"` +} + // Collection represents a defradb collection. // // A Collection is mostly analogous to a SQL table, however a collection is specific to its @@ -23,17 +31,20 @@ import ( // // Many functions on this object will interact with the underlying datastores. type Collection interface { - // Description returns the CollectionDescription of this Collection. - Description() CollectionDescription // Name returns the name of this collection. Name() string - // Schema returns the SchemaDescription used to define this Collection. - Schema() SchemaDescription // ID returns the ID of this Collection. ID() uint32 // SchemaID returns the ID of the Schema used to define this Collection. SchemaID() string + // Definition contains the metadata defining what a Collection is. + Definition() CollectionDefinition + // Schema returns the SchemaDescription used to define this Collection. + Schema() SchemaDescription + // Description returns the CollectionDescription of this Collection. + Description() CollectionDescription + // Create a new document. // // Will verify the DocKey/CID to ensure that the new document is correctly formatted. diff --git a/client/descriptions.go b/client/descriptions.go index 4f388fa7d3..8dbe54ddce 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -28,6 +28,8 @@ type CollectionDescription struct { ID uint32 // Schema contains the data type information that this Collection uses. + // + // This property is deprecated and should not be used. Schema SchemaDescription // Indexes contains the secondary indexes that this Collection has. @@ -41,12 +43,21 @@ func (col CollectionDescription) IDString() string { // GetFieldByID searches for a field with the given ID. If such a field is found it // will return it and true, if it is not found it will return false. -func (col CollectionDescription) GetFieldByID(id FieldID) (FieldDescription, bool) { - if !col.Schema.IsEmpty() { - for _, field := range col.Schema.Fields { - if field.ID == id { - return field, true - } +func (col CollectionDescription) GetFieldByID(id FieldID, schema *SchemaDescription) (FieldDescription, bool) { + for _, field := range schema.Fields { + if field.ID == id { + return field, true + } + } + return FieldDescription{}, false +} + +// GetFieldByName returns the field for the given field name. If such a field is found it +// will return it and true, if it is not found it will return false. +func (col CollectionDescription) GetFieldByName(fieldName string, schema *SchemaDescription) (FieldDescription, bool) { + for _, field := range schema.Fields { + if field.Name == fieldName { + return field, true } } return FieldDescription{}, false @@ -57,8 +68,9 @@ func (col CollectionDescription) GetFieldByRelation( relationName string, otherCollectionName string, otherFieldName string, + schema *SchemaDescription, ) (FieldDescription, bool) { - for _, field := range col.Schema.Fields { + for _, field := range schema.Fields { if field.RelationName == relationName && !(col.Name == otherCollectionName && otherFieldName == field.Name) { return field, true } @@ -93,28 +105,11 @@ type SchemaDescription struct { Fields []FieldDescription } -// IsEmpty returns true if the SchemaDescription is empty and uninitialized -func (sd SchemaDescription) IsEmpty() bool { - return len(sd.Fields) == 0 -} - -// GetFieldKey returns the field ID for the given field name. -func (sd SchemaDescription) GetFieldKey(fieldName string) uint32 { - for _, field := range sd.Fields { - if field.Name == fieldName { - return uint32(field.ID) - } - } - return uint32(0) -} - // GetField returns the field of the given name. func (sd SchemaDescription) GetField(name string) (FieldDescription, bool) { - if !sd.IsEmpty() { - for _, field := range sd.Fields { - if field.Name == name { - return field, true - } + for _, field := range sd.Fields { + if field.Name == name { + return field, true } } return FieldDescription{}, false diff --git a/core/parser.go b/core/parser.go index ee2d2cfbf1..300f4411a4 100644 --- a/core/parser.go +++ b/core/parser.go @@ -51,8 +51,8 @@ type Parser interface { NewFilterFromString(collectionType string, body string) (immutable.Option[request.Filter], error) // ParseSDL parses an SDL string into a set of collection descriptions. - ParseSDL(ctx context.Context, schemaString string) ([]client.CollectionDescription, error) + ParseSDL(ctx context.Context, schemaString string) ([]client.CollectionDefinition, error) // Adds the given schema to this parser's model. - SetSchema(ctx context.Context, txn datastore.Txn, collections []client.CollectionDescription) error + SetSchema(ctx context.Context, txn datastore.Txn, collections []client.CollectionDefinition) error } diff --git a/db/base/collection_keys.go b/db/base/collection_keys.go index 0276975630..6a762ff180 100644 --- a/db/base/collection_keys.go +++ b/db/base/collection_keys.go @@ -34,6 +34,7 @@ func MakeDocKey(col client.CollectionDescription, docKey string) core.DataStoreK func MakePrimaryIndexKeyForCRDT( c client.CollectionDescription, + schema client.SchemaDescription, ctype client.CType, key core.DataStoreKey, fieldName string, @@ -42,19 +43,12 @@ func MakePrimaryIndexKeyForCRDT( case client.COMPOSITE: return MakeCollectionKey(c).WithInstanceInfo(key).WithFieldId(core.COMPOSITE_NAMESPACE), nil case client.LWW_REGISTER: - fieldKey := getFieldKey(c, key, fieldName) - return MakeCollectionKey(c).WithInstanceInfo(fieldKey), nil - } - return core.DataStoreKey{}, ErrInvalidCrdtType -} + field, ok := c.GetFieldByName(fieldName, &schema) + if !ok { + return core.DataStoreKey{}, client.NewErrFieldNotExist(fieldName) + } -func getFieldKey( - c client.CollectionDescription, - key core.DataStoreKey, - fieldName string, -) core.DataStoreKey { - if !c.Schema.IsEmpty() { - return key.WithFieldId(fmt.Sprint(c.Schema.GetFieldKey(fieldName))) + return MakeCollectionKey(c).WithInstanceInfo(key).WithFieldId(fmt.Sprint(field.ID)), nil } - return key.WithFieldId(fieldName) + return core.DataStoreKey{}, ErrInvalidCrdtType } diff --git a/db/collection.go b/db/collection.go index df8ca85cc1..8fdf9089ed 100644 --- a/db/collection.go +++ b/db/collection.go @@ -54,11 +54,7 @@ type collection struct { // of the operation in question. txn immutable.Option[datastore.Txn] - colID uint32 - - schemaID string - - desc client.CollectionDescription + def client.CollectionDefinition indexes []CollectionIndex fetcherFactory func() fetcher.Fetcher @@ -71,42 +67,10 @@ type collection struct { // CollectionOptions object. // NewCollection returns a pointer to a newly instanciated DB Collection -func (db *db) newCollection(desc client.CollectionDescription) (*collection, error) { - if desc.Name == "" { - return nil, client.NewErrUninitializeProperty("Collection", "Name") - } - - if len(desc.Schema.Fields) == 0 { - return nil, client.NewErrUninitializeProperty("Collection", "Fields") - } - - docKeyField := desc.Schema.Fields[0] - if docKeyField.Kind != client.FieldKind_DocKey || docKeyField.Name != request.KeyFieldName { - return nil, ErrSchemaFirstFieldDocKey - } - - for i, field := range desc.Schema.Fields { - if field.Name == "" { - return nil, client.NewErrUninitializeProperty("Collection.Schema", "Name") - } - if field.Kind == client.FieldKind_None { - return nil, client.NewErrUninitializeProperty("Collection.Schema", "FieldKind") - } - if (field.Kind != client.FieldKind_DocKey && !field.IsObject()) && - field.Typ == client.NONE_CRDT { - return nil, client.NewErrUninitializeProperty("Collection.Schema", "CRDT type") - } - desc.Schema.Fields[i].ID = client.FieldID(i) - } - +func (db *db) newCollection(desc client.CollectionDescription, schema client.SchemaDescription) (*collection, error) { return &collection{ - db: db, - desc: client.CollectionDescription{ - ID: desc.ID, - Name: desc.Name, - Schema: desc.Schema, - }, - colID: desc.ID, + db: db, + def: client.CollectionDefinition{Description: desc, Schema: schema}, }, nil } @@ -130,8 +94,11 @@ func (c *collection) newFetcher() fetcher.Fetcher { func (db *db) createCollection( ctx context.Context, txn datastore.Txn, - desc client.CollectionDescription, + def client.CollectionDefinition, ) (client.Collection, error) { + schema := def.Schema + desc := def.Description + // check if collection by this name exists collectionKey := core.NewCollectionKey(desc.Name) exists, err := txn.Systemstore().Has(ctx, collectionKey.ToDS()) @@ -151,14 +118,19 @@ func (db *db) createCollection( return nil, err } desc.ID = uint32(colID) - col, err := db.newCollection(desc) + + for i := range schema.Fields { + schema.Fields[i].ID = client.FieldID(i) + } + + col, err := db.newCollection(desc, schema) if err != nil { return nil, err } // Local elements such as secondary indexes should be excluded // from the (global) schemaId. - schemaBuf, err := json.Marshal(col.desc.Schema) + schemaBuf, err := json.Marshal(schema) if err != nil { return nil, err } @@ -169,16 +141,16 @@ func (db *db) createCollection( return nil, err } schemaID := cid.String() - col.schemaID = schemaID // For new schemas the initial version id will match the schema id schemaVersionID := schemaID - col.desc.Schema.VersionID = schemaVersionID - col.desc.Schema.SchemaID = schemaID + schema.VersionID = schemaVersionID + schema.SchemaID = schemaID + desc.Schema = schema // buffer must include all the ids, as it is saved and loaded from the store later. - buf, err := json.Marshal(col.desc) + buf, err := json.Marshal(desc) if err != nil { return nil, err } @@ -214,7 +186,8 @@ func (db *db) createCollection( return nil, err } } - return col, nil + + return db.getCollectionByName(ctx, txn, desc.Name) } // updateCollection updates the persisted collection description matching the name of the given @@ -229,24 +202,40 @@ func (db *db) updateCollection( ctx context.Context, txn datastore.Txn, existingDescriptionsByName map[string]client.CollectionDescription, - proposedDescriptionsByName map[string]client.CollectionDescription, - desc client.CollectionDescription, + existingSchemaByName map[string]client.SchemaDescription, + proposedDescriptionsByName map[string]client.SchemaDescription, + def client.CollectionDefinition, setAsDefaultVersion bool, ) (client.Collection, error) { - hasChanged, err := db.validateUpdateCollection(ctx, txn, existingDescriptionsByName, proposedDescriptionsByName, desc) + schema := def.Schema + desc := def.Description + + hasChanged, err := db.validateUpdateCollection(ctx, existingDescriptionsByName, desc) + if err != nil { + return nil, err + } + + hasSchemaChanged, err := db.validateUpdateSchema( + ctx, + txn, + existingSchemaByName, + proposedDescriptionsByName, + schema, + ) if err != nil { return nil, err } + hasChanged = hasChanged || hasSchemaChanged if !hasChanged { return db.getCollectionByName(ctx, txn, desc.Name) } - for _, field := range desc.Schema.Fields { + for _, field := range schema.Fields { if field.RelationType.IsSet(client.Relation_Type_ONE) { idFieldName := field.Name + "_id" - if _, ok := desc.Schema.GetField(idFieldName); !ok { - desc.Schema.Fields = append(desc.Schema.Fields, client.FieldDescription{ + if _, ok := schema.GetField(idFieldName); !ok { + schema.Fields = append(schema.Fields, client.FieldDescription{ Name: idFieldName, Kind: client.FieldKind_DocKey, RelationType: client.Relation_Type_INTERNAL_ID, @@ -256,23 +245,23 @@ func (db *db) updateCollection( } } - for i, field := range desc.Schema.Fields { + for i, field := range schema.Fields { if field.ID == client.FieldID(0) { // This is not wonderful and will probably break when we add the ability // to delete fields, however it is good enough for now and matches the // create behaviour. field.ID = client.FieldID(i) - desc.Schema.Fields[i] = field + schema.Fields[i] = field } if field.Typ == client.NONE_CRDT { // If no CRDT Type has been provided, default to LWW_REGISTER. field.Typ = client.LWW_REGISTER - desc.Schema.Fields[i] = field + schema.Fields[i] = field } } - globalSchemaBuf, err := json.Marshal(desc.Schema) + globalSchemaBuf, err := json.Marshal(schema) if err != nil { return nil, err } @@ -281,9 +270,10 @@ func (db *db) updateCollection( if err != nil { return nil, err } - previousSchemaVersionID := desc.Schema.VersionID + previousSchemaVersionID := schema.VersionID schemaVersionID := cid.String() - desc.Schema.VersionID = schemaVersionID + schema.VersionID = schemaVersionID + desc.Schema = schema buf, err := json.Marshal(desc) if err != nil { @@ -298,14 +288,14 @@ func (db *db) updateCollection( return nil, err } - schemaVersionHistoryKey := core.NewSchemaHistoryKey(desc.Schema.SchemaID, previousSchemaVersionID) + schemaVersionHistoryKey := core.NewSchemaHistoryKey(schema.SchemaID, previousSchemaVersionID) err = txn.Systemstore().Put(ctx, schemaVersionHistoryKey.ToDS(), []byte(schemaVersionID)) if err != nil { return nil, err } if setAsDefaultVersion { - err = db.setDefaultSchemaVersionExplicit(ctx, txn, desc.Name, desc.Schema.SchemaID, schemaVersionID) + err = db.setDefaultSchemaVersionExplicit(ctx, txn, desc.Name, schema.SchemaID, schemaVersionID) if err != nil { return nil, err } @@ -320,9 +310,7 @@ func (db *db) updateCollection( // collection. Will return an error if it fails validation. func (db *db) validateUpdateCollection( ctx context.Context, - txn datastore.Txn, existingDescriptionsByName map[string]client.CollectionDescription, - proposedDescriptionsByName map[string]client.CollectionDescription, proposedDesc client.CollectionDescription, ) (bool, error) { if proposedDesc.Name == "" { @@ -338,50 +326,73 @@ func (db *db) validateUpdateCollection( return false, NewErrCollectionIDDoesntMatch(proposedDesc.Name, existingDesc.ID, proposedDesc.ID) } - if proposedDesc.Schema.SchemaID != existingDesc.Schema.SchemaID { + hasChangedIndexes, err := validateUpdateCollectionIndexes(existingDesc.Indexes, proposedDesc.Indexes) + return hasChangedIndexes, err +} + +// validateUpdateSchema validates that the given schema description is a valid update. +// +// Will return true if the given description differs from the current persisted state of the +// schema. Will return an error if it fails validation. +func (db *db) validateUpdateSchema( + ctx context.Context, + txn datastore.Txn, + existingDescriptionsByName map[string]client.SchemaDescription, + proposedDescriptionsByName map[string]client.SchemaDescription, + proposedDesc client.SchemaDescription, +) (bool, error) { + if proposedDesc.Name == "" { + return false, ErrSchemaNameEmpty + } + + existingDesc, collectionExists := existingDescriptionsByName[proposedDesc.Name] + if !collectionExists { + return false, NewErrAddCollectionWithPatch(proposedDesc.Name) + } + + if proposedDesc.SchemaID != existingDesc.SchemaID { return false, NewErrSchemaIDDoesntMatch( proposedDesc.Name, - existingDesc.Schema.SchemaID, - proposedDesc.Schema.SchemaID, + existingDesc.SchemaID, + proposedDesc.SchemaID, ) } - if proposedDesc.Schema.Name != existingDesc.Schema.Name { + if proposedDesc.Name != existingDesc.Name { // There is actually little reason to not support this atm besides controlling the surface area // of the new feature. Changing this should not break anything, but it should be tested first. - return false, NewErrCannotModifySchemaName(existingDesc.Schema.Name, proposedDesc.Schema.Name) + return false, NewErrCannotModifySchemaName(existingDesc.Name, proposedDesc.Name) } - if proposedDesc.Schema.VersionID != "" && proposedDesc.Schema.VersionID != existingDesc.Schema.VersionID { + if proposedDesc.VersionID != "" && proposedDesc.VersionID != existingDesc.VersionID { // If users specify this it will be overwritten, an error is prefered to quietly ignoring it. return false, ErrCannotSetVersionID } - hasChangedFields, err := validateUpdateCollectionFields(proposedDescriptionsByName, existingDesc, proposedDesc) + hasChangedFields, err := validateUpdateSchemaFields(proposedDescriptionsByName, existingDesc, proposedDesc) if err != nil { return hasChangedFields, err } - hasChangedIndexes, err := validateUpdateCollectionIndexes(existingDesc.Indexes, proposedDesc.Indexes) - return hasChangedFields || hasChangedIndexes, err + return hasChangedFields, err } -func validateUpdateCollectionFields( - descriptionsByName map[string]client.CollectionDescription, - existingDesc client.CollectionDescription, - proposedDesc client.CollectionDescription, +func validateUpdateSchemaFields( + descriptionsByName map[string]client.SchemaDescription, + existingDesc client.SchemaDescription, + proposedDesc client.SchemaDescription, ) (bool, error) { hasChanged := false existingFieldsByID := map[client.FieldID]client.FieldDescription{} existingFieldIndexesByName := map[string]int{} - for i, field := range existingDesc.Schema.Fields { + for i, field := range existingDesc.Fields { existingFieldIndexesByName[field.Name] = i existingFieldsByID[field.ID] = field } newFieldNames := map[string]struct{}{} newFieldIds := map[client.FieldID]struct{}{} - for proposedIndex, proposedField := range proposedDesc.Schema.Fields { + for proposedIndex, proposedField := range proposedDesc.Fields { var existingField client.FieldDescription var fieldAlreadyExists bool if proposedField.ID != client.FieldID(0) || @@ -449,7 +460,7 @@ func validateUpdateCollectionFields( if proposedField.Kind == client.FieldKind_FOREIGN_OBJECT { idFieldName := proposedField.Name + request.RelatedObjectID - idField, idFieldFound := proposedDesc.Schema.GetField(idFieldName) + idField, idFieldFound := proposedDesc.GetField(idFieldName) if idFieldFound { if idField.Kind != client.FieldKind_DocKey { return false, NewErrRelationalFieldIDInvalidType(idField.Name, client.FieldKind_DocKey, idField.Kind) @@ -471,7 +482,7 @@ func validateUpdateCollectionFields( var relatedFieldFound bool var relatedField client.FieldDescription - for _, field := range relatedDesc.Schema.Fields { + for _, field := range relatedDesc.Fields { if field.RelationName == proposedField.RelationName && !field.RelationType.IsSet(client.Relation_Type_INTERNAL_ID) && !(relatedDesc.Name == proposedDesc.Name && field.Name == proposedField.Name) { @@ -545,7 +556,7 @@ func validateUpdateCollectionFields( newFieldIds[proposedField.ID] = struct{}{} } - for _, field := range existingDesc.Schema.Fields { + for _, field := range existingDesc.Fields { if _, stillExists := newFieldIds[field.ID]; !stillExists { return false, NewErrCannotDeleteField(field.Name, field.ID) } @@ -600,12 +611,17 @@ func (db *db) setDefaultSchemaVersion( return err } - cols, err := db.getCollectionDescriptions(ctx, txn) + cols, err := db.getAllCollections(ctx, txn) if err != nil { return err } - return db.parser.SetSchema(ctx, txn, cols) + definitions := make([]client.CollectionDefinition, len(cols)) + for i, col := range cols { + definitions[i] = col.Definition() + } + + return db.parser.SetSchema(ctx, txn, definitions) } func (db *db) setDefaultSchemaVersionExplicit( @@ -650,10 +666,11 @@ func (db *db) getCollectionByVersionID( } col := &collection{ - db: db, - desc: desc, - colID: desc.ID, - schemaID: desc.Schema.SchemaID, + db: db, + def: client.CollectionDefinition{ + Description: desc, + Schema: desc.Schema, + }, } err = col.loadIndexes(ctx, txn) @@ -751,7 +768,7 @@ func (c *collection) getAllDocKeysChan( txn datastore.Txn, ) (<-chan client.DocKeysResult, error) { prefix := core.PrimaryDataStoreKey{ // empty path for all keys prefix - CollectionId: fmt.Sprint(c.colID), + CollectionId: fmt.Sprint(c.ID()), } q, err := txn.Datastore().Query(ctx, query.Query{ Prefix: prefix.ToString(), @@ -806,26 +823,30 @@ func (c *collection) getAllDocKeysChan( // Description returns the client.CollectionDescription. func (c *collection) Description() client.CollectionDescription { - return c.desc + return c.Definition().Description } // Name returns the collection name. func (c *collection) Name() string { - return c.desc.Name + return c.Description().Name } // Schema returns the Schema of the collection. func (c *collection) Schema() client.SchemaDescription { - return c.desc.Schema + return c.Definition().Schema } // ID returns the ID of the collection. func (c *collection) ID() uint32 { - return c.colID + return c.Description().ID } func (c *collection) SchemaID() string { - return c.schemaID + return c.Schema().SchemaID +} + +func (c *collection) Definition() client.CollectionDefinition { + return c.def } // WithTxn returns a new instance of the collection, with a transaction @@ -834,9 +855,7 @@ func (c *collection) WithTxn(txn datastore.Txn) client.Collection { return &collection{ db: c.db, txn: immutable.Some(txn), - desc: c.desc, - colID: c.colID, - schemaID: c.schemaID, + def: c.def, indexes: c.indexes, fetcherFactory: c.fetcherFactory, } @@ -894,7 +913,7 @@ func (c *collection) getKeysFromDoc( func (c *collection) create(ctx context.Context, txn datastore.Txn, doc *client.Document) error { // This has to be done before dockey verification happens in the next step. - if err := doc.RemapAliasFieldsAndDockey(c.desc.Schema.Fields); err != nil { + if err := doc.RemapAliasFieldsAndDockey(c.Schema().Fields); err != nil { return err } @@ -1050,7 +1069,7 @@ func (c *collection) save( return cid.Undef, client.NewErrFieldNotExist(k) } - fieldDescription, valid := c.desc.Schema.GetField(k) + fieldDescription, valid := c.Schema().GetField(k) if !valid { return cid.Undef, client.NewErrFieldNotExist(k) } @@ -1121,7 +1140,7 @@ func (c *collection) save( events.Update{ DocKey: doc.Key().String(), Cid: headNode.Cid(), - SchemaID: c.schemaID, + SchemaID: c.Schema().SchemaID, Block: headNode, Priority: priority, }, @@ -1152,7 +1171,7 @@ func (c *collection) validateOneToOneLinkDoesntAlreadyExist( return nil } - objFieldDescription, ok := c.desc.Schema.GetField(strings.TrimSuffix(fieldDescription.Name, request.RelatedObjectID)) + objFieldDescription, ok := c.Schema().GetField(strings.TrimSuffix(fieldDescription.Name, request.RelatedObjectID)) if !ok { return client.NewErrFieldNotExist(strings.TrimSuffix(fieldDescription.Name, request.RelatedObjectID)) } @@ -1320,10 +1339,17 @@ func (c *collection) saveValueToMerkleCRDT( if err != nil { return nil, 0, err } - field, _ := c.Description().GetFieldByID(client.FieldID(fieldID)) + + schema := c.Schema() + + field, ok := c.Description().GetFieldByID(client.FieldID(fieldID), &schema) + if !ok { + return nil, 0, client.NewErrFieldIndexNotExist(fieldID) + } + merkleCRDT, err := c.db.crdtFactory.InstanceWithStores( txn, - core.NewCollectionSchemaVersionKey(c.Schema().VersionID), + core.NewCollectionSchemaVersionKey(schema.VersionID), c.db.events.Updates, ctype, key, @@ -1334,7 +1360,6 @@ func (c *collection) saveValueToMerkleCRDT( } var bytes []byte - var ok bool // parse args if len(args) != 1 { return nil, 0, ErrUnknownCRDTArgument @@ -1417,14 +1442,14 @@ func (c *collection) commitImplicitTxn(ctx context.Context, txn datastore.Txn) e func (c *collection) getPrimaryKeyFromDocKey(docKey client.DocKey) core.PrimaryDataStoreKey { return core.PrimaryDataStoreKey{ - CollectionId: fmt.Sprint(c.colID), + CollectionId: fmt.Sprint(c.ID()), DocKey: docKey.String(), } } func (c *collection) getDSKeyFromDockey(docKey client.DocKey) core.DataStoreKey { return core.DataStoreKey{ - CollectionID: fmt.Sprint(c.colID), + CollectionID: fmt.Sprint(c.ID()), DocKey: docKey.String(), InstanceType: core.ValueKey, } @@ -1446,7 +1471,7 @@ func (c *collection) tryGetFieldKey(key core.PrimaryDataStoreKey, fieldName stri // tryGetSchemaFieldID returns the FieldID of the given fieldName. // Will return false if the field is not found. func (c *collection) tryGetSchemaFieldID(fieldName string) (uint32, bool) { - for _, field := range c.desc.Schema.Fields { + for _, field := range c.Schema().Fields { if field.Name == fieldName { if field.IsObject() || field.IsObjectArray() { // We do not wish to match navigational properties, only diff --git a/db/collection_delete.go b/db/collection_delete.go index 480656849f..a24eb496f6 100644 --- a/db/collection_delete.go +++ b/db/collection_delete.go @@ -207,7 +207,7 @@ func (c *collection) deleteWithFilter( // Convert from string to client.DocKey. key := core.PrimaryDataStoreKey{ - CollectionId: fmt.Sprint(c.colID), + CollectionId: fmt.Sprint(c.ID()), DocKey: docKey, } @@ -281,7 +281,7 @@ func (c *collection) applyDelete( events.Update{ DocKey: key.DocKey, Cid: headNode.Cid(), - SchemaID: c.schemaID, + SchemaID: c.Schema().SchemaID, Block: headNode, Priority: priority, }, diff --git a/db/collection_get.go b/db/collection_get.go index 8262ff44ba..d210072793 100644 --- a/db/collection_get.go +++ b/db/collection_get.go @@ -53,16 +53,15 @@ func (c *collection) get( ) (*client.Document, error) { // create a new document fetcher df := c.newFetcher() - desc := &c.desc // initialize it with the primary index - err := df.Init(ctx, txn, &c.desc, fields, nil, nil, false, showDeleted) + err := df.Init(ctx, txn, c, fields, nil, nil, false, showDeleted) if err != nil { _ = df.Close() return nil, err } // construct target key for DocKey - targetKey := base.MakeDocKey(*desc, key.DocKey) + targetKey := base.MakeDocKey(c.Description(), key.DocKey) // run the doc fetcher err = df.Start(ctx, core.NewSpans(core.NewSpan(targetKey, targetKey.PrefixEnd()))) if err != nil { diff --git a/db/collection_index.go b/db/collection_index.go index f3c1ba2e98..278586902b 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -121,10 +121,11 @@ func (c *collection) updateIndexedDoc( return err } desc := c.Description() + schema := c.Schema() oldDoc, err := c.get( ctx, txn, - c.getPrimaryKeyFromDocKey(doc.Key()), desc.CollectIndexedFields(&desc.Schema), + c.getPrimaryKeyFromDocKey(doc.Key()), desc.CollectIndexedFields(&schema), false, ) if err != nil { @@ -217,7 +218,7 @@ func (c *collection) createIndex( if err != nil { return nil, err } - c.desc.Indexes = append(c.desc.Indexes, colIndex.Description()) + c.def.Description.Indexes = append(c.def.Description.Indexes, colIndex.Description()) c.indexes = append(c.indexes, colIndex) err = c.indexExistingDocs(ctx, txn, colIndex) if err != nil { @@ -233,12 +234,12 @@ func (c *collection) iterateAllDocs( exec func(doc *client.Document) error, ) error { df := c.newFetcher() - err := df.Init(ctx, txn, &c.desc, fields, nil, nil, false, false) + err := df.Init(ctx, txn, c, fields, nil, nil, false, false) if err != nil { _ = df.Close() return err } - start := base.MakeCollectionKey(c.desc) + start := base.MakeCollectionKey(c.Description()) spans := core.NewSpans(core.NewSpan(start, start.PrefixEnd())) err = df.Start(ctx, spans) @@ -278,8 +279,8 @@ func (c *collection) indexExistingDocs( ) error { fields := make([]client.FieldDescription, 0, 1) for _, field := range index.Description().Fields { - for i := range c.desc.Schema.Fields { - colField := c.desc.Schema.Fields[i] + for i := range c.Schema().Fields { + colField := c.Schema().Fields[i] if field.Name == colField.Name { fields = append(fields, colField) break @@ -333,9 +334,9 @@ func (c *collection) dropIndex(ctx context.Context, txn datastore.Txn, indexName return NewErrIndexWithNameDoesNotExists(indexName) } - for i := range c.desc.Indexes { - if c.desc.Indexes[i].Name == indexName { - c.desc.Indexes = append(c.desc.Indexes[:i], c.desc.Indexes[i+1:]...) + for i := range c.Description().Indexes { + if c.Description().Indexes[i].Name == indexName { + c.def.Description.Indexes = append(c.Description().Indexes[:i], c.Description().Indexes[i+1:]...) break } } @@ -379,7 +380,7 @@ func (c *collection) loadIndexes(ctx context.Context, txn datastore.Txn) error { } colIndexes = append(colIndexes, index) } - c.desc.Indexes = indexDescriptions + c.def.Description.Indexes = indexDescriptions c.indexes = colIndexes return nil } @@ -396,14 +397,14 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, if err != nil { return nil, err } - return c.desc.Indexes, nil + return c.Description().Indexes, nil } func (c *collection) checkExistingFields( ctx context.Context, fields []client.IndexedFieldDescription, ) error { - collectionFields := c.Description().Schema.Fields + collectionFields := c.Schema().Fields for _, field := range fields { found := false for _, colField := range collectionFields { diff --git a/db/collection_test.go b/db/collection_test.go index e3686504d3..dd57cb285b 100644 --- a/db/collection_test.go +++ b/db/collection_test.go @@ -12,320 +12,11 @@ package db import ( "context" - "reflect" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/sourcenetwork/defradb/client" ) -func newTestCollectionWithSchema( - t *testing.T, - ctx context.Context, - db *implicitTxnDB, -) (client.Collection, error) { - desc := client.CollectionDescription{ - Name: "users", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "Name", - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - { - Name: "Age", - Kind: client.FieldKind_INT, - Typ: client.LWW_REGISTER, - }, - { - Name: "Weight", - Kind: client.FieldKind_FLOAT, - Typ: client.LWW_REGISTER, - }, - }, - }, - } - - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - col, err := db.createCollection(ctx, txn, desc) - if err != nil { - return col, err - } - - return col, txn.Commit(ctx) -} - -func TestNewCollection_ReturnsError_GivenNoSchema(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - _, err = db.createCollection(ctx, txn, client.CollectionDescription{ - Name: "test", - }) - assert.Error(t, err) -} - -func TestNewCollectionWithSchema(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - schema := col.Schema() - desc := col.Description() - - assert.True(t, reflect.DeepEqual(schema, desc.Schema)) - assert.Equal(t, "users", col.Name()) - assert.Equal(t, uint32(1), col.ID()) - assert.False(t, reflect.DeepEqual(schema, client.SchemaDescription{})) - assert.Equal(t, 4, len(schema.Fields)) - - for i := 0; i < 4; i++ { - assert.Equal(t, client.FieldID(i), schema.Fields[i].ID) - } -} - -func TestNewCollectionReturnsErrorGivenDuplicateSchema(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - - _, err = newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - _, err = newTestCollectionWithSchema(t, ctx, db) - assert.Errorf(t, err, "collection already exists") -} - -func TestNewCollectionReturnsErrorGivenNoFields(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - desc := client.CollectionDescription{ - Name: "users", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{}, - }, - } - - _, err = db.createCollection(ctx, txn, desc) - assert.EqualError( - t, - err, - "invalid state, required property is uninitialized. Host: Collection, PropertyName: Fields", - ) -} - -func TestNewCollectionReturnsErrorGivenNoName(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - desc := client.CollectionDescription{ - Name: "", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{}, - }, - } - - _, err = db.createCollection(ctx, txn, desc) - assert.EqualError( - t, - err, - "invalid state, required property is uninitialized. Host: Collection, PropertyName: Name", - ) -} - -func TestNewCollectionReturnsErrorGivenNoKeyField(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - desc := client.CollectionDescription{ - Name: "users", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "Name", - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - }, - }, - } - - _, err = db.createCollection(ctx, txn, desc) - assert.EqualError(t, err, "collection schema first field must be a DocKey") -} - -func TestNewCollectionReturnsErrorGivenKeyFieldIsNotFirstField(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - desc := client.CollectionDescription{ - Name: "users", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "Name", - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - }, - }, - } - - _, err = db.createCollection(ctx, txn, desc) - assert.EqualError(t, err, "collection schema first field must be a DocKey") -} - -func TestNewCollectionReturnsErrorGivenFieldWithNoName(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - desc := client.CollectionDescription{ - Name: "users", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "", - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - }, - }, - } - - _, err = db.createCollection(ctx, txn, desc) - assert.EqualError( - t, - err, - "invalid state, required property is uninitialized. Host: Collection.Schema, PropertyName: Name", - ) -} - -func TestNewCollectionReturnsErrorGivenFieldWithNoKind(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - desc := client.CollectionDescription{ - Name: "users", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "Name", - Typ: client.LWW_REGISTER, - }, - }, - }, - } - - _, err = db.createCollection(ctx, txn, desc) - assert.EqualError( - t, - err, - "invalid state, required property is uninitialized. Host: Collection.Schema, PropertyName: FieldKind", - ) -} - -func TestNewCollectionReturnsErrorGivenFieldWithNoType(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - txn, err := db.NewTxn(ctx, false) - require.NoError(t, err) - - desc := client.CollectionDescription{ - Name: "users", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "Name", - Kind: client.FieldKind_STRING, - }, - }, - }, - } - - _, err = db.createCollection(ctx, txn, desc) - assert.EqualError( - t, - err, - "invalid state, required property is uninitialized. Host: Collection.Schema, PropertyName: CRDT type", - ) -} - -func TestGetCollectionByName(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - - _, err = newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - col, err := db.GetCollectionByName(ctx, "users") - assert.NoError(t, err) - - schema := col.Schema() - desc := col.Description() - - assert.True(t, reflect.DeepEqual(schema, desc.Schema)) - assert.Equal(t, "users", col.Name()) - assert.Equal(t, uint32(1), col.ID()) - assert.False(t, reflect.DeepEqual(schema, client.SchemaDescription{})) - assert.Equal(t, 4, len(schema.Fields)) - - for i := 0; i < 4; i++ { - assert.Equal(t, client.FieldID(i), schema.Fields[i].ID) - } -} - func TestGetCollectionByNameReturnsErrorGivenNonExistantCollection(t *testing.T) { ctx := context.Background() db, err := newMemoryDB(ctx) diff --git a/db/collection_update.go b/db/collection_update.go index 2e353dd0d3..c68902db44 100644 --- a/db/collection_update.go +++ b/db/collection_update.go @@ -303,13 +303,13 @@ func (c *collection) applyMergeToDoc( }) for mfield, mval := range mergeMap { - fd, isValidField := c.desc.Schema.GetField(mfield) + fd, isValidField := c.Schema().GetField(mfield) if !isValidField { return client.NewErrFieldNotExist(mfield) } if fd.Kind == client.FieldKind_FOREIGN_OBJECT { - fd, isValidField = c.desc.Schema.GetField(mfield + request.RelatedObjectID) + fd, isValidField = c.Schema().GetField(mfield + request.RelatedObjectID) if !isValidField { return client.NewErrFieldNotExist(mfield) } @@ -335,7 +335,7 @@ func (c *collection) isSecondaryIDField(fieldDesc client.FieldDescription) (clie return client.FieldDescription{}, false } - relationFieldDescription, valid := c.Description().Schema.GetField( + relationFieldDescription, valid := c.Schema().GetField( strings.TrimSuffix(fieldDesc.Name, request.RelatedObjectID), ) return relationFieldDescription, valid && !relationFieldDescription.IsPrimaryRelation() @@ -365,17 +365,19 @@ func (c *collection) patchPrimaryDoc( return err } primaryCol = primaryCol.WithTxn(txn) + primarySchema := primaryCol.Schema() primaryField, ok := primaryCol.Description().GetFieldByRelation( relationFieldDescription.RelationName, secondaryCollectionName, relationFieldDescription.Name, + &primarySchema, ) if !ok { return client.NewErrFieldNotExist(relationFieldDescription.RelationName) } - primaryIDField, ok := primaryCol.Description().Schema.GetField(primaryField.Name + request.RelatedObjectID) + primaryIDField, ok := primaryCol.Schema().GetField(primaryField.Name + request.RelatedObjectID) if !ok { return client.NewErrFieldNotExist(primaryField.Name + request.RelatedObjectID) } diff --git a/db/db_test.go b/db/db_test.go index c1a9648f36..9d681058c8 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -15,14 +15,8 @@ import ( "testing" badger "github.com/dgraph-io/badger/v4" - dag "github.com/ipfs/boxo/ipld/merkledag" - "github.com/stretchr/testify/assert" - "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/core" - corecrdt "github.com/sourcenetwork/defradb/core/crdt" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" - "github.com/sourcenetwork/defradb/merkle/clock" ) func newMemoryDB(ctx context.Context) (*implicitTxnDB, error) { @@ -48,369 +42,3 @@ func TestNewDB(t *testing.T) { t.Error(err) } } - -func TestDBSaveSimpleDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21, - "Weight": 154.1 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - if err != nil { - t.Error(err) - return - } - - err = col.Save(ctx, doc) - if err != nil { - t.Error(err) - } - - // value check - name, err := doc.Get("Name") - assert.NoError(t, err) - age, err := doc.Get("Age") - assert.NoError(t, err) - weight, err := doc.Get("Weight") - assert.NoError(t, err) - - assert.Equal(t, "John", name) - assert.Equal(t, int64(21), age) - assert.Equal(t, 154.1, weight) - - _, err = doc.Get("DoesntExist") - assert.Error(t, err) - - // db.printDebugDB() -} - -func TestDBUpdateDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21, - "Weight": 154.1 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - if err != nil { - t.Error(err) - return - } - - err = col.Save(ctx, doc) - if err != nil { - t.Error(err) - } - - // update fields - doc.Set("Name", "Pete") - doc.Delete("Weight") - - weightField := doc.Fields()["Weight"] - weightVal, _ := doc.GetValueWithField(weightField) - assert.True(t, weightVal.IsDelete()) - - err = col.Update(ctx, doc) - if err != nil { - t.Error(err) - } - - // value check - name, err := doc.Get("Name") - assert.NoError(t, err) - age, err := doc.Get("Age") - assert.NoError(t, err) - weight, err := doc.Get("Weight") - assert.NoError(t, err) - - assert.Equal(t, "Pete", name) - assert.Equal(t, int64(21), age) - assert.Nil(t, weight) -} - -func TestDBUpdateNonExistingDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21, - "Weight": 154.1 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - if err != nil { - t.Error(err) - return - } - - err = col.Update(ctx, doc) - assert.Error(t, err) -} - -func TestDBUpdateExistingDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21, - "Weight": 154.1 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - assert.NoError(t, err) - - err = col.Save(ctx, doc) - assert.NoError(t, err) - - testJSONObj = []byte(`{ - "_key": "bae-09cd7539-9b86-5661-90f6-14fbf6c1a14d", - "Name": "Pete", - "Age": 31 - }`) - - doc, err = client.NewDocFromJSON(testJSONObj) - assert.NoError(t, err) - - err = col.Update(ctx, doc) - assert.NoError(t, err) - - // value check - name, err := doc.Get("Name") - assert.NoError(t, err) - age, err := doc.Get("Age") - assert.NoError(t, err) - // weight, err := doc.Get("Weight") - // assert.NoError(t, err) - - assert.Equal(t, "Pete", name) - assert.Equal(t, int64(31), age) -} - -func TestDBGetDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21, - "Weight": 154.1 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - assert.NoError(t, err) - - err = col.Save(ctx, doc) - assert.NoError(t, err) - - key, err := client.NewDocKeyFromString("bae-09cd7539-9b86-5661-90f6-14fbf6c1a14d") - assert.NoError(t, err) - doc, err = col.Get(ctx, key, false) - assert.NoError(t, err) - - // value check - name, err := doc.Get("Name") - assert.NoError(t, err) - age, err := doc.Get("Age") - assert.NoError(t, err) - weight, err := doc.Get("Weight") - assert.NoError(t, err) - - assert.Equal(t, "John", name) - assert.Equal( - t, - uint64(21), - age, - ) // note: uint is used here, because the CBOR implementation converts all positive ints to uint64 - assert.Equal(t, 154.1, weight) -} - -func TestDBGetNotFoundDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - key, err := client.NewDocKeyFromString("bae-09cd7539-9b86-5661-90f6-14fbf6c1a14d") - assert.NoError(t, err) - _, err = col.Get(ctx, key, false) - assert.EqualError(t, err, client.ErrDocumentNotFound.Error()) -} - -func TestDBDeleteDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21, - "Weight": 154.1 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - assert.NoError(t, err) - - err = col.Save(ctx, doc) - assert.NoError(t, err) - - key, err := client.NewDocKeyFromString("bae-09cd7539-9b86-5661-90f6-14fbf6c1a14d") - assert.NoError(t, err) - deleted, err := col.Delete(ctx, key) - assert.NoError(t, err) - assert.True(t, deleted) -} - -func TestDBDeleteNotFoundDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - key, err := client.NewDocKeyFromString("bae-09cd7539-9b86-5661-90f6-14fbf6c1a14d") - assert.NoError(t, err) - deleted, err := col.Delete(ctx, key) - assert.EqualError(t, err, client.ErrDocumentNotFound.Error()) - assert.False(t, deleted) -} - -func TestDocumentMerkleDAG(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21, - "Weight": 154.1 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - assert.NoError(t, err) - - err = col.Save(ctx, doc) - assert.NoError(t, err) - - clk := clock.NewMerkleClock( - db.multistore.Headstore(), - nil, - core.HeadStoreKey{}.WithDocKey( - "bae-09cd7539-9b86-5661-90f6-14fbf6c1a14d", - ).WithFieldId( - "Name", - ), - nil, - ) - heads := clk.(*clock.MerkleClock).Heads() - cids, _, err := heads.List(ctx) - assert.NoError(t, err) - - reg := corecrdt.LWWRegister{} - for _, c := range cids { - b, errGet := db.Blockstore().Get(ctx, c) - assert.NoError(t, errGet) - - nd, errDecode := dag.DecodeProtobuf(b.RawData()) - assert.NoError(t, errDecode) - - _, errMarshal := nd.MarshalJSON() - assert.NoError(t, errMarshal) - - _, errDeltaDecode := reg.DeltaDecode(nd) - assert.NoError(t, errDeltaDecode) - } - - testJSONObj = []byte(`{ - "_key": "bae-09cd7539-9b86-5661-90f6-14fbf6c1a14d", - "Name": "Pete", - "Age": 31 - }`) - - doc, err = client.NewDocFromJSON(testJSONObj) - assert.NoError(t, err) - - err = col.Update(ctx, doc) - assert.NoError(t, err) - - heads = clk.(*clock.MerkleClock).Heads() - cids, _, err = heads.List(ctx) - assert.NoError(t, err) - - for _, c := range cids { - b, err := db.Blockstore().Get(ctx, c) - assert.NoError(t, err) - - nd, err := dag.DecodeProtobuf(b.RawData()) - assert.NoError(t, err) - - _, err = nd.MarshalJSON() - assert.NoError(t, err) - - _, err = reg.DeltaDecode(nd) - assert.NoError(t, err) - } -} - -// collection with schema -func TestDBSchemaSaveSimpleDocument(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - testJSONObj := []byte(`{ - "Name": "John", - "Age": 21 - }`) - - doc, err := client.NewDocFromJSON(testJSONObj) - if err != nil { - t.Error(err) - return - } - - err = col.Save(ctx, doc) - assert.NoError(t, err) - - // value check - name, err := doc.Get("Name") - assert.NoError(t, err) - age, err := doc.Get("Age") - assert.NoError(t, err) - - assert.Equal(t, "John", name) - assert.Equal(t, int64(21), age) - - err = db.PrintDump(ctx) - assert.Nil(t, err) -} diff --git a/db/errors.go b/db/errors.go index 4a456cd41a..d4e883c11e 100644 --- a/db/errors.go +++ b/db/errors.go @@ -111,9 +111,9 @@ var ( ErrDocumentDeleted = errors.New(errDocumentDeleted) ErrUnknownCRDTArgument = errors.New("invalid CRDT arguments") ErrUnknownCRDT = errors.New("unknown crdt") - ErrSchemaFirstFieldDocKey = errors.New("collection schema first field must be a DocKey") ErrCollectionAlreadyExists = errors.New("collection already exists") ErrCollectionNameEmpty = errors.New("collection name can't be empty") + ErrSchemaNameEmpty = errors.New("schema name can't be empty") ErrSchemaIDEmpty = errors.New("schema ID can't be empty") ErrSchemaVersionIDEmpty = errors.New("schema version ID can't be empty") ErrKeyEmpty = errors.New("key cannot be empty") diff --git a/db/fetcher/fetcher.go b/db/fetcher/fetcher.go index 8935e617cc..da7a0df1e1 100644 --- a/db/fetcher/fetcher.go +++ b/db/fetcher/fetcher.go @@ -57,7 +57,7 @@ type Fetcher interface { Init( ctx context.Context, txn datastore.Txn, - col *client.CollectionDescription, + col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, @@ -81,7 +81,7 @@ var ( // DocumentFetcher is a utility to incrementally fetch all the documents. type DocumentFetcher struct { - col *client.CollectionDescription + col client.Collection reverse bool deletedDocs bool @@ -137,7 +137,7 @@ type DocumentFetcher struct { func (df *DocumentFetcher) Init( ctx context.Context, txn datastore.Txn, - col *client.CollectionDescription, + col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, @@ -145,9 +145,6 @@ func (df *DocumentFetcher) Init( showDeleted bool, ) error { df.txn = txn - if col.Schema.IsEmpty() { - return client.NewErrUninitializeProperty("DocumentFetcher", "Schema") - } err := df.init(col, fields, filter, docmapper, reverse) if err != nil { @@ -166,7 +163,7 @@ func (df *DocumentFetcher) Init( } func (df *DocumentFetcher) init( - col *client.CollectionDescription, + col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docMapper *core.DocumentMapping, @@ -202,7 +199,7 @@ func (df *DocumentFetcher) init( // get them all var targetFields []client.FieldDescription if len(fields) == 0 { - targetFields = df.col.Schema.Fields + targetFields = df.col.Schema().Fields } else { targetFields = fields } @@ -213,12 +210,12 @@ func (df *DocumentFetcher) init( if df.filter != nil { conditions := df.filter.ToMap(df.mapping) - parsedfilterFields, err := parser.ParseFilterFieldsForDescription(conditions, df.col.Schema) + parsedfilterFields, err := parser.ParseFilterFieldsForDescription(conditions, df.col.Schema()) if err != nil { return err } df.filterFields = make(map[uint32]client.FieldDescription, len(parsedfilterFields)) - df.filterSet = bitset.New(uint(len(col.Schema.Fields))) + df.filterSet = bitset.New(uint(len(col.Schema().Fields))) for _, field := range parsedfilterFields { df.filterFields[uint32(field.ID)] = field df.filterSet.Set(uint(field.ID)) @@ -253,7 +250,7 @@ func (df *DocumentFetcher) start(ctx context.Context, spans core.Spans, withDele df.deletedDocs = withDeleted if !spans.HasValue { // no specified spans so create a prefix scan key for the entire collection - start := base.MakeCollectionKey(*df.col) + start := base.MakeCollectionKey(df.col.Description()) if withDeleted { start = start.WithDeletedFlag() } else { diff --git a/db/fetcher/indexer.go b/db/fetcher/indexer.go index da4dc6a580..a0ee94d0b9 100644 --- a/db/fetcher/indexer.go +++ b/db/fetcher/indexer.go @@ -24,7 +24,7 @@ import ( // It fetches only the indexed field and the rest of the fields are fetched by the internal fetcher. type IndexFetcher struct { docFetcher Fetcher - col *client.CollectionDescription + col client.Collection txn datastore.Txn indexFilter *mapper.Filter docFilter *mapper.Filter @@ -55,7 +55,7 @@ func NewIndexFetcher( func (f *IndexFetcher) Init( ctx context.Context, txn datastore.Txn, - col *client.CollectionDescription, + col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docMapper *core.DocumentMapping, @@ -68,14 +68,14 @@ func (f *IndexFetcher) Init( f.mapping = docMapper f.txn = txn - for _, index := range col.Indexes { + for _, index := range col.Description().Indexes { if index.Fields[0].Name == f.indexedField.Name { f.indexDataStoreKey.IndexID = index.ID break } } - f.indexDataStoreKey.CollectionID = f.col.ID + f.indexDataStoreKey.CollectionID = f.col.ID() for i := range fields { if fields[i].Name == f.indexedField.Name { @@ -131,7 +131,7 @@ func (f *IndexFetcher) FetchNext(ctx context.Context) (EncodedDocument, ExecInfo f.execInfo.FieldsFetched++ if f.docFetcher != nil && len(f.docFields) > 0 { - targetKey := base.MakeDocKey(*f.col, string(f.doc.key)) + targetKey := base.MakeDocKey(f.col.Description(), string(f.doc.key)) spans := core.NewSpans(core.NewSpan(targetKey, targetKey.PrefixEnd())) err = f.docFetcher.Start(ctx, spans) if err != nil { diff --git a/db/fetcher/mocks/fetcher.go b/db/fetcher/mocks/fetcher.go index 79eefefc2b..1597b13b2e 100644 --- a/db/fetcher/mocks/fetcher.go +++ b/db/fetcher/mocks/fetcher.go @@ -134,11 +134,11 @@ func (_c *Fetcher_FetchNext_Call) RunAndReturn(run func(context.Context) (fetche } // Init provides a mock function with given fields: ctx, txn, col, fields, filter, docmapper, reverse, showDeleted -func (_m *Fetcher) Init(ctx context.Context, txn datastore.Txn, col *client.CollectionDescription, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, reverse bool, showDeleted bool) error { +func (_m *Fetcher) Init(ctx context.Context, txn datastore.Txn, col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, reverse bool, showDeleted bool) error { ret := _m.Called(ctx, txn, col, fields, filter, docmapper, reverse, showDeleted) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, datastore.Txn, *client.CollectionDescription, []client.FieldDescription, *mapper.Filter, *core.DocumentMapping, bool, bool) error); ok { + if rf, ok := ret.Get(0).(func(context.Context, datastore.Txn, client.Collection, []client.FieldDescription, *mapper.Filter, *core.DocumentMapping, bool, bool) error); ok { r0 = rf(ctx, txn, col, fields, filter, docmapper, reverse, showDeleted) } else { r0 = ret.Error(0) @@ -155,7 +155,7 @@ type Fetcher_Init_Call struct { // Init is a helper method to define mock.On call // - ctx context.Context // - txn datastore.Txn -// - col *client.CollectionDescription +// - col client.Collection // - fields []client.FieldDescription // - filter *mapper.Filter // - docmapper *core.DocumentMapping @@ -165,9 +165,9 @@ func (_e *Fetcher_Expecter) Init(ctx interface{}, txn interface{}, col interface return &Fetcher_Init_Call{Call: _e.mock.On("Init", ctx, txn, col, fields, filter, docmapper, reverse, showDeleted)} } -func (_c *Fetcher_Init_Call) Run(run func(ctx context.Context, txn datastore.Txn, col *client.CollectionDescription, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, reverse bool, showDeleted bool)) *Fetcher_Init_Call { +func (_c *Fetcher_Init_Call) Run(run func(ctx context.Context, txn datastore.Txn, col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, reverse bool, showDeleted bool)) *Fetcher_Init_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(datastore.Txn), args[2].(*client.CollectionDescription), args[3].([]client.FieldDescription), args[4].(*mapper.Filter), args[5].(*core.DocumentMapping), args[6].(bool), args[7].(bool)) + run(args[0].(context.Context), args[1].(datastore.Txn), args[2].(client.Collection), args[3].([]client.FieldDescription), args[4].(*mapper.Filter), args[5].(*core.DocumentMapping), args[6].(bool), args[7].(bool)) }) return _c } @@ -177,7 +177,7 @@ func (_c *Fetcher_Init_Call) Return(_a0 error) *Fetcher_Init_Call { return _c } -func (_c *Fetcher_Init_Call) RunAndReturn(run func(context.Context, datastore.Txn, *client.CollectionDescription, []client.FieldDescription, *mapper.Filter, *core.DocumentMapping, bool, bool) error) *Fetcher_Init_Call { +func (_c *Fetcher_Init_Call) RunAndReturn(run func(context.Context, datastore.Txn, client.Collection, []client.FieldDescription, *mapper.Filter, *core.DocumentMapping, bool, bool) error) *Fetcher_Init_Call { _c.Call.Return(run) return _c } diff --git a/db/fetcher/versioned.go b/db/fetcher/versioned.go index f1c7b6a9de..da670b1c27 100644 --- a/db/fetcher/versioned.go +++ b/db/fetcher/versioned.go @@ -92,7 +92,7 @@ type VersionedFetcher struct { queuedCids *list.List - col *client.CollectionDescription + col client.Collection // @todo index *client.IndexDescription mCRDTs map[uint32]crdt.MerkleCRDT } @@ -101,7 +101,7 @@ type VersionedFetcher struct { func (vf *VersionedFetcher) Init( ctx context.Context, txn datastore.Txn, - col *client.CollectionDescription, + col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, @@ -357,13 +357,14 @@ func (vf *VersionedFetcher) merge(c cid.Cid) error { return err } - fieldID := vf.col.Schema.GetFieldKey(l.Name) - if fieldID == uint32(0) { + schema := vf.col.Schema() + field, ok := vf.col.Description().GetFieldByName(l.Name, &schema) + if !ok { return client.NewErrFieldNotExist(l.Name) } // @todo: Right now we ONLY handle LWW_REGISTER, need to swith on this and // get CType from descriptions - if err := vf.processNode(fieldID, subNd, client.LWW_REGISTER, l.Name); err != nil { + if err := vf.processNode(uint32(field.ID), subNd, client.LWW_REGISTER, l.Name); err != nil { return err } } @@ -380,7 +381,7 @@ func (vf *VersionedFetcher) processNode( // handle CompositeDAG mcrdt, exists := vf.mCRDTs[crdtIndex] if !exists { - key, err := base.MakePrimaryIndexKeyForCRDT(*vf.col, ctype, vf.key, fieldName) + key, err := base.MakePrimaryIndexKeyForCRDT(vf.col.Description(), vf.col.Schema(), ctype, vf.key, fieldName) if err != nil { return err } diff --git a/db/fetcher_test.go b/db/fetcher_test.go index e2c3647792..f7de9bf036 100644 --- a/db/fetcher_test.go +++ b/db/fetcher_test.go @@ -16,169 +16,13 @@ import ( "github.com/stretchr/testify/assert" - "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" - "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/db/base" "github.com/sourcenetwork/defradb/db/fetcher" ) -func newTestCollectionDescription() client.CollectionDescription { - return client.CollectionDescription{ - Name: "users", - ID: uint32(1), - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - ID: client.FieldID(1), - Kind: client.FieldKind_DocKey, - }, - { - Name: "Name", - ID: client.FieldID(2), - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - { - Name: "Age", - ID: client.FieldID(3), - Kind: client.FieldKind_INT, - Typ: client.LWW_REGISTER, - }, - }, - }, - } -} - -func newTestFetcher(ctx context.Context, txn datastore.Txn) (*fetcher.DocumentFetcher, error) { - df := new(fetcher.DocumentFetcher) - desc := newTestCollectionDescription() - err := df.Init(ctx, txn, &desc, desc.Schema.Fields, nil, nil, false, false) - if err != nil { - return nil, err - } - return df, nil -} - -func TestFetcherInit(t *testing.T) { - _, err := newTestFetcher(context.Background(), nil) - assert.NoError(t, err) -} - -func TestFetcherStart(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - if err != nil { - t.Error(err) - return - } - txn, err := db.NewTxn(ctx, true) - if err != nil { - t.Error(err) - return - } - df, err := newTestFetcher(ctx, txn) - assert.NoError(t, err) - - err = df.Start(ctx, core.Spans{}) - assert.NoError(t, err) -} - func TestFetcherStartWithoutInit(t *testing.T) { ctx := context.Background() df := new(fetcher.DocumentFetcher) err := df.Start(ctx, core.Spans{}) assert.Error(t, err) } - -func TestMakeIndexPrefixKey(t *testing.T) { - desc := newTestCollectionDescription() - key := base.MakeCollectionKey(desc) - assert.Equal(t, "/1", key.ToString()) -} - -func TestFetcherGetAllPrimaryIndexEncodedDocSingle(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - doc, err := client.NewDocFromJSON([]byte(`{ - "Name": "John", - "Age": 21 - }`)) - assert.NoError(t, err) - err = col.Save(ctx, doc) - assert.NoError(t, err) - - txn, err := db.NewTxn(ctx, true) - if err != nil { - t.Error(err) - return - } - - // db.printDebugDB() - - df := new(fetcher.DocumentFetcher) - desc := col.Description() - err = df.Init(ctx, txn, &desc, desc.Schema.Fields, nil, nil, false, false) - assert.NoError(t, err) - - err = df.Start(ctx, core.Spans{}) - assert.NoError(t, err) - - encdoc, _, err := df.FetchNext(ctx) - assert.NoError(t, err) - assert.NotNil(t, encdoc) -} - -func TestFetcherGetAllPrimaryIndexEncodedDocMultiple(t *testing.T) { - ctx := context.Background() - db, err := newMemoryDB(ctx) - assert.NoError(t, err) - - col, err := newTestCollectionWithSchema(t, ctx, db) - assert.NoError(t, err) - - doc, err := client.NewDocFromJSON([]byte(`{ - "Name": "John", - "Age": 21 - }`)) - assert.NoError(t, err) - err = col.Save(ctx, doc) - assert.NoError(t, err) - - doc, err = client.NewDocFromJSON([]byte(`{ - "Name": "Alice", - "Age": 27 - }`)) - assert.NoError(t, err) - err = col.Save(ctx, doc) - assert.NoError(t, err) - - txn, err := db.NewTxn(ctx, true) - if err != nil { - t.Error(err) - return - } - - // db.printDebugDB() - - df := new(fetcher.DocumentFetcher) - desc := col.Description() - err = df.Init(ctx, txn, &desc, desc.Schema.Fields, nil, nil, false, false) - assert.NoError(t, err) - - err = df.Start(ctx, core.Spans{}) - assert.NoError(t, err) - - encdoc, _, err := df.FetchNext(ctx) - assert.NoError(t, err) - assert.NotNil(t, encdoc) - encdoc, _, err = df.FetchNext(ctx) - assert.NoError(t, err) - assert.NotNil(t, encdoc) -} diff --git a/db/index.go b/db/index.go index 7314bc2a08..e1aaed6cb6 100644 --- a/db/index.go +++ b/db/index.go @@ -82,10 +82,8 @@ func NewCollectionIndex( return nil, NewErrIndexDescHasNoFields(desc) } index := &collectionSimpleIndex{collection: collection, desc: desc} - schema := collection.Description().Schema - fieldID := client.FieldID(schema.GetFieldKey(desc.Fields[0].Name)) - field, foundField := collection.Description().GetFieldByID(fieldID) - if fieldID == client.FieldID(0) || !foundField { + field, foundField := collection.Schema().GetField(desc.Fields[0].Name) + if !foundField { return nil, NewErrIndexDescHasNonExistingField(desc, desc.Fields[0].Name) } var e error diff --git a/db/index_test.go b/db/index_test.go index 67c3f232d0..d22746a363 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -15,6 +15,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "strings" "testing" ds "github.com/ipfs/go-datastore" @@ -55,71 +56,62 @@ type indexTestFixture struct { ctx context.Context db *implicitTxnDB txn datastore.Txn - users *collection + users client.Collection t *testing.T } -func getUsersCollectionDesc() client.CollectionDescription { - return client.CollectionDescription{ - Name: usersColName, - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: usersNameFieldName, - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - { - Name: usersAgeFieldName, - Kind: client.FieldKind_INT, - Typ: client.LWW_REGISTER, - }, - { - Name: usersWeightFieldName, - Kind: client.FieldKind_FLOAT, - Typ: client.LWW_REGISTER, - }, - }, - }, - } -} +func (f *indexTestFixture) getUsersCollectionDesc() client.Collection { + _, err := f.db.AddSchema( + f.ctx, + fmt.Sprintf( + `type %s { + %s: String + %s: Int + %s: Float + }`, + usersColName, + usersNameFieldName, + usersAgeFieldName, + usersWeightFieldName, + ), + ) + require.NoError(f.t, err) -func getProductsCollectionDesc() client.CollectionDescription { - return client.CollectionDescription{ - Name: productsColName, - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: productsIDFieldName, - Kind: client.FieldKind_INT, - Typ: client.LWW_REGISTER, - }, - { - Name: productsPriceFieldName, - Kind: client.FieldKind_FLOAT, - Typ: client.LWW_REGISTER, - }, - { - Name: productsCategoryFieldName, - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - { - Name: productsAvailableFieldName, - Kind: client.FieldKind_BOOL, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + col, err := f.db.GetCollectionByName(f.ctx, usersColName) + require.NoError(f.t, err) + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + + return col +} + +func (f *indexTestFixture) getProductsCollectionDesc() client.Collection { + _, err := f.db.AddSchema( + f.ctx, + fmt.Sprintf( + `type %s { + %s: Int + %s: Float + %s: String + %s: Boolean + }`, + productsColName, + productsIDFieldName, + productsPriceFieldName, + productsCategoryFieldName, + productsAvailableFieldName, + ), + ) + require.NoError(f.t, err) + + col, err := f.db.GetCollectionByName(f.ctx, productsColName) + require.NoError(f.t, err) + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + + return col } func newIndexTestFixtureBare(t *testing.T) *indexTestFixture { @@ -139,7 +131,7 @@ func newIndexTestFixtureBare(t *testing.T) *indexTestFixture { func newIndexTestFixture(t *testing.T) *indexTestFixture { f := newIndexTestFixtureBare(t) - f.users = f.createCollection(getUsersCollectionDesc()) + f.users = f.getUsersCollectionDesc() return f } @@ -247,18 +239,6 @@ func (f *indexTestFixture) getCollectionIndexes(colName string) ([]client.IndexD return f.db.fetchCollectionIndexDescriptions(f.ctx, f.txn, colName) } -func (f *indexTestFixture) createCollection( - desc client.CollectionDescription, -) *collection { - col, err := f.db.createCollection(f.ctx, f.txn, desc) - assert.NoError(f.t, err) - err = f.txn.Commit(f.ctx) - assert.NoError(f.t, err) - f.txn, err = f.db.NewTxn(f.ctx, false) - assert.NoError(f.t, err) - return col.(*collection) -} - func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { f := newIndexTestFixture(t) @@ -324,28 +304,6 @@ func TestCreateIndex_IfFieldHasNoDirection_DefaultToAsc(t *testing.T) { assert.Equal(t, client.Ascending, newDesc.Fields[0].Direction) } -func TestCreateIndex_IfNameIsNotSpecified_Generate(t *testing.T) { - f := newIndexTestFixtureBare(t) - colDesc := getUsersCollectionDesc() - const colName = "UsErS" - const fieldName = "NaMe" - colDesc.Name = colName - colDesc.Schema.Name = colName // Which one should we use? - colDesc.Schema.Fields[1].Name = fieldName - f.users = f.createCollection(colDesc) - - desc := client.IndexDescription{ - Name: "", - Fields: []client.IndexedFieldDescription{ - {Name: fieldName, Direction: client.Ascending}, - }, - } - - newDesc, err := f.createCollectionIndex(desc) - assert.NoError(t, err) - assert.Equal(t, colName+"_"+fieldName+"_ASC", newDesc.Name) -} - func TestCreateIndex_IfSingleFieldInDescOrder_ReturnError(t *testing.T) { f := newIndexTestFixture(t) @@ -515,8 +473,8 @@ func TestCreateIndex_IfPropertyDoesntExist_ReturnError(t *testing.T) { func TestCreateIndex_WithMultipleCollectionsAndIndexes_AssignIncrementedIDPerCollection(t *testing.T) { f := newIndexTestFixtureBare(t) - users := f.createCollection(getUsersCollectionDesc()) - products := f.createCollection(getProductsCollectionDesc()) + users := f.getUsersCollectionDesc() + products := f.getProductsCollectionDesc() makeIndex := func(fieldName string) client.IndexDescription { return client.IndexDescription{ @@ -606,24 +564,16 @@ func TestCreateIndex_IfAttemptToIndexOnUnsupportedType_ReturnError(t *testing.T) const unsupportedKind = client.FieldKind_BOOL_ARRAY - desc := client.CollectionDescription{ - Name: "testTypeCol", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "field", - Kind: unsupportedKind, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + _, err := f.db.AddSchema( + f.ctx, + `type testTypeCol { + field: [Boolean!] + }`, + ) + require.NoError(f.t, err) - collection := f.createCollection(desc) + collection, err := f.db.GetCollectionByName(f.ctx, "testTypeCol") + require.NoError(f.t, err) indexDesc := client.IndexDescription{ Fields: []client.IndexedFieldDescription{ @@ -631,7 +581,10 @@ func TestCreateIndex_IfAttemptToIndexOnUnsupportedType_ReturnError(t *testing.T) }, } - _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + + _, err = f.createCollectionIndexFor(collection.Name(), indexDesc) require.ErrorIs(f.t, err, NewErrUnsupportedIndexFieldType(unsupportedKind)) f.commitTxn() } @@ -652,7 +605,7 @@ func TestCreateIndex_IfFailedToReadIndexUponRetrievingCollectionDesc_ReturnError onSystemStore.Query(mock.Anything, mock.MatchedBy(matchPrefixFunc)).Return(nil, testErr) - descData, err := json.Marshal(getUsersCollectionDesc()) + descData, err := json.Marshal(f.users.Description()) require.NoError(t, err) onSystemStore.Query(mock.Anything, mock.Anything). @@ -676,7 +629,9 @@ func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) assert.NoError(t, err) - f.createCollection(getProductsCollectionDesc()) + f.commitTxn() + + f.getProductsCollectionDesc() productsIndexDesc := client.IndexDescription{ Name: "products_description_index", Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, @@ -830,11 +785,17 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) assert.NoError(t, err) - f.createCollection(getProductsCollectionDesc()) + f.commitTxn() + + f.getProductsCollectionDesc() productsIndexDesc := client.IndexDescription{ Name: "products_description_index", Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, } + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) assert.NoError(t, err) @@ -1021,27 +982,23 @@ func TestCollectionGetIndexes_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { func TestCollectionGetIndexes_IfStoredIndexWithUnsupportedType_ReturnError(t *testing.T) { f := newIndexTestFixtureBare(t) + f.getUsersCollectionDesc() const unsupportedKind = client.FieldKind_BOOL_ARRAY + _, err := f.db.AddSchema( + f.ctx, + `type testTypeCol { + name: String + field: [Boolean!] + }`, + ) + require.NoError(f.t, err) - desc := client.CollectionDescription{ - Name: "testTypeCol", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "field", - Kind: unsupportedKind, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + collection, err := f.db.GetCollectionByName(f.ctx, "testTypeCol") + require.NoError(f.t, err) - collection := f.createCollection(desc) + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) indexDesc := client.IndexDescription{ Fields: []client.IndexedFieldDescription{ @@ -1121,17 +1078,6 @@ func TestCollectionGetIndexes_IfIndexIsDropped_ReturnUpdateIndexes(t *testing.T) func TestCollectionGetIndexes_ShouldReturnIndexesInOrderedByName(t *testing.T) { f := newIndexTestFixtureBare(t) - colDesc := client.CollectionDescription{ - Name: "testCollection", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - }, - }, - } const ( num = 30 fieldNamePrefix = "field_" @@ -1142,17 +1088,33 @@ func TestCollectionGetIndexes_ShouldReturnIndexesInOrderedByName(t *testing.T) { return fmt.Sprintf("%02d", i) } + builder := strings.Builder{} + builder.WriteString("type testCollection {\n") + for i := 1; i <= num; i++ { - colDesc.Schema.Fields = append(colDesc.Schema.Fields, - client.FieldDescription{ - Name: fieldNamePrefix + toSuffix(i), - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }) + _, err := builder.WriteString(fieldNamePrefix) + require.NoError(f.t, err) + + _, err = builder.WriteString(toSuffix(i)) + require.NoError(f.t, err) + + _, err = builder.WriteString(": String\n") + require.NoError(f.t, err) } + _, err := builder.WriteString("}") + require.NoError(f.t, err) - collection := f.createCollection(colDesc) + _, err = f.db.AddSchema( + f.ctx, + builder.String(), + ) + require.NoError(f.t, err) + collection, err := f.db.GetCollectionByName(f.ctx, "testCollection") + require.NoError(f.t, err) + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) for i := 1; i <= num; i++ { iStr := toSuffix(i) indexDesc := client.IndexDescription{ @@ -1319,7 +1281,7 @@ func TestDropAllIndexes_ShouldDeleteAllIndexes(t *testing.T) { assert.Equal(t, 2, f.countIndexPrefixes(usersColName, "")) - err = f.users.dropAllIndexes(f.ctx, f.txn) + err = f.users.(*collection).dropAllIndexes(f.ctx, f.txn) assert.NoError(t, err) assert.Equal(t, 0, f.countIndexPrefixes(usersColName, "")) @@ -1331,7 +1293,7 @@ func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { f.db.Close(f.ctx) - err := f.users.dropAllIndexes(f.ctx, f.txn) + err := f.users.(*collection).dropAllIndexes(f.ctx, f.txn) assert.Error(t, err) } @@ -1386,7 +1348,7 @@ func TestDropAllIndexes_IfSystemStorageFails_ReturnError(t *testing.T) { mockedTxn.EXPECT().Systemstore().Unset() mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() - err := f.users.dropAllIndexes(f.ctx, f.txn) + err := f.users.(*collection).dropAllIndexes(f.ctx, f.txn) assert.ErrorIs(t, err, testErr, testCase.Name) } } @@ -1406,7 +1368,7 @@ func TestDropAllIndexes_ShouldCloseQueryIterator(t *testing.T) { mockedTxn.EXPECT().Systemstore().Unset() mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() - _ = f.users.dropAllIndexes(f.ctx, f.txn) + _ = f.users.(*collection).dropAllIndexes(f.ctx, f.txn) } func TestNewCollectionIndex_IfDescriptionHasNoFields_ReturnError(t *testing.T) { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index 5634686778..5b25fab21a 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -234,11 +234,9 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E systemStoreOn.Get(mock.Anything, colKey.ToDS()).Maybe().Return([]byte(userColVersionID), nil) colVersionIDKey := core.NewCollectionSchemaVersionKey(userColVersionID) - colDesc := getUsersCollectionDesc() - colDesc.ID = 1 - for i := range colDesc.Schema.Fields { - colDesc.Schema.Fields[i].ID = client.FieldID(i) - } + usersCol, err := f.db.GetCollectionByName(f.ctx, usersColName) + require.NoError(f.t, err) + colDesc := usersCol.Description() colDescBytes, err := json.Marshal(colDesc) require.NoError(f.t, err) systemStoreOn.Get(mock.Anything, colVersionIDKey.ToDS()).Maybe().Return(colDescBytes, nil) @@ -361,8 +359,8 @@ func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { func TestNonUnique_IfMultipleCollectionsWithIndexes_StoreIndexWithCollectionID(t *testing.T) { f := newIndexTestFixtureBare(t) - users := f.createCollection(getUsersCollectionDesc()) - products := f.createCollection(getProductsCollectionDesc()) + users := f.getUsersCollectionDesc() + products := f.getProductsCollectionDesc() _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) require.NoError(f.t, err) @@ -437,24 +435,23 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { } for i, tc := range testCase { - desc := client.CollectionDescription{ - Name: "testTypeCol" + strconv.Itoa(i), - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "field", - Kind: tc.FieldKind, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + _, err := f.db.AddSchema( + f.ctx, + fmt.Sprintf( + `type %s { + field: %s + }`, + "testTypeCol"+strconv.Itoa(i), + tc.FieldKind.String(), + ), + ) + require.NoError(f.t, err) + + collection, err := f.db.GetCollectionByName(f.ctx, "testTypeCol"+strconv.Itoa(i)) + require.NoError(f.t, err) - collection := f.createCollection(desc) + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) indexDesc := client.IndexDescription{ Fields: []client.IndexedFieldDescription{ @@ -462,7 +459,7 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { }, } - _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + _, err = f.createCollectionIndexFor(collection.Name(), indexDesc) require.NoError(f.t, err) f.commitTxn() @@ -596,7 +593,7 @@ func TestNonUniqueCreate_IfUponIndexingExistingDocsFetcherFails_ReturnError(t *t doc := f.newUserDoc("John", 21) f.saveDocToCollection(doc, f.users) - f.users.fetcherFactory = tc.PrepareFetcher + f.users.(*collection).fetcherFactory = tc.PrepareFetcher key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() _, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) @@ -614,7 +611,7 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) f.saveDocToCollection(doc, f.users) fieldKeyString := core.DataStoreKey{ - CollectionID: f.users.desc.IDString(), + CollectionID: f.users.Description().IDString(), }.WithDocKey(doc.Key().String()). WithFieldId("1"). WithValueFlag(). @@ -623,7 +620,7 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) invalidKeyString := fieldKeyString + "/doesn't matter/" // Insert an invalid key within the document prefix, this will generate an error within the fetcher. - f.users.db.multistore.Datastore().Put(f.ctx, ipfsDatastore.NewKey(invalidKeyString), []byte("doesn't matter")) + f.db.multistore.Datastore().Put(f.ctx, ipfsDatastore.NewKey(invalidKeyString), []byte("doesn't matter")) _, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) require.ErrorIs(f.t, err, core.ErrInvalidKey) @@ -631,7 +628,7 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f := newIndexTestFixtureBare(t) - users := f.createCollection(getUsersCollectionDesc()) + users := f.getUsersCollectionDesc() _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) require.NoError(f.t, err) _, err = f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnAge()) @@ -643,7 +640,7 @@ func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f.saveDocToCollection(f.newUserDoc("John", 21), users) f.saveDocToCollection(f.newUserDoc("Islam", 23), users) - products := f.createCollection(getProductsCollectionDesc()) + products := f.getProductsCollectionDesc() _, err = f.createCollectionIndexFor(products.Name(), getProductsIndexDescOnCategory()) require.NoError(f.t, err) f.commitTxn() @@ -885,7 +882,7 @@ func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveDocToCollection(doc, f.users) - f.users.fetcherFactory = tc.PrepareFetcher + f.users.(*collection).fetcherFactory = tc.PrepareFetcher oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() err := doc.Set(usersNameFieldName, "Islam") @@ -931,14 +928,14 @@ func TestNonUniqueUpdate_ShouldPassToFetcherOnlyRelevantFields(t *testing.T) { f.createUserCollectionIndexOnName() f.createUserCollectionIndexOnAge() - f.users.fetcherFactory = func() fetcher.Fetcher { + f.users.(*collection).fetcherFactory = func() fetcher.Fetcher { f := fetcherMocks.NewStubbedFetcher(t) f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Unset() f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). RunAndReturn(func( ctx context.Context, txn datastore.Txn, - col *client.CollectionDescription, + col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, mapping *core.DocumentMapping, @@ -999,7 +996,7 @@ func TestNonUniqueUpdate_IfDatastoreFails_ReturnError(t *testing.T) { schemaVersionID: f.users.Schema().VersionID, } - f.users.fetcherFactory = func() fetcher.Fetcher { + f.users.(*collection).fetcherFactory = func() fetcher.Fetcher { df := fetcherMocks.NewStubbedFetcher(t) df.EXPECT().FetchNext(mock.Anything).Unset() df.EXPECT().FetchNext(mock.Anything).Return(&encodedDoc, fetcher.ExecInfo{}, nil) diff --git a/db/p2p_collection_test.go b/db/p2p_collection_test.go index acd80bd041..67d5393c66 100644 --- a/db/p2p_collection_test.go +++ b/db/p2p_collection_test.go @@ -12,6 +12,7 @@ package db import ( "context" + "fmt" "testing" "github.com/stretchr/testify/require" @@ -25,31 +26,18 @@ func newTestCollection( db *implicitTxnDB, name string, ) client.Collection { - desc := client.CollectionDescription{ - Name: name, - Schema: client.SchemaDescription{ - Name: name, - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "Name", - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - }, - }, - } - - txn, err := db.db.NewTxn(ctx, false) + _, err := db.AddSchema( + ctx, + fmt.Sprintf( + `type %s { + Name: String + }`, + name, + ), + ) require.NoError(t, err) - col, err := db.db.createCollection(ctx, txn, desc) - require.NoError(t, err) - - err = txn.Commit(ctx) + col, err := db.GetCollectionByName(ctx, name) require.NoError(t, err) return col diff --git a/db/schema.go b/db/schema.go index 910f44f8c1..ec14393563 100644 --- a/db/schema.go +++ b/db/schema.go @@ -39,24 +39,29 @@ func (db *db) addSchema( txn datastore.Txn, schemaString string, ) ([]client.CollectionDescription, error) { - existingDescriptions, err := db.getCollectionDescriptions(ctx, txn) + existingCollections, err := db.getAllCollections(ctx, txn) if err != nil { return nil, err } - newDescriptions, err := db.parser.ParseSDL(ctx, schemaString) + existingDefinitions := make([]client.CollectionDefinition, len(existingCollections)) + for i := range existingCollections { + existingDefinitions[i] = existingCollections[i].Definition() + } + + newDefinitions, err := db.parser.ParseSDL(ctx, schemaString) if err != nil { return nil, err } - err = db.parser.SetSchema(ctx, txn, append(existingDescriptions, newDescriptions...)) + err = db.parser.SetSchema(ctx, txn, append(existingDefinitions, newDefinitions...)) if err != nil { return nil, err } - returnDescriptions := make([]client.CollectionDescription, len(newDescriptions)) - for i, desc := range newDescriptions { - col, err := db.createCollection(ctx, txn, desc) + returnDescriptions := make([]client.CollectionDescription, len(newDefinitions)) + for i, definition := range newDefinitions { + col, err := db.createCollection(ctx, txn, definition) if err != nil { return nil, err } @@ -67,29 +72,17 @@ func (db *db) addSchema( } func (db *db) loadSchema(ctx context.Context, txn datastore.Txn) error { - descriptions, err := db.getCollectionDescriptions(ctx, txn) - if err != nil { - return err - } - - return db.parser.SetSchema(ctx, txn, descriptions) -} - -func (db *db) getCollectionDescriptions( - ctx context.Context, - txn datastore.Txn, -) ([]client.CollectionDescription, error) { collections, err := db.getAllCollections(ctx, txn) if err != nil { - return nil, err + return err } - descriptions := make([]client.CollectionDescription, len(collections)) - for i, collection := range collections { - descriptions[i] = collection.Description() + definitions := make([]client.CollectionDefinition, len(collections)) + for i := range collections { + definitions[i] = collections[i].Definition() } - return descriptions, nil + return db.parser.SetSchema(ctx, txn, definitions) } // patchSchema takes the given JSON patch string and applies it to the set of CollectionDescriptions @@ -114,6 +107,11 @@ func (db *db) patchSchema(ctx context.Context, txn datastore.Txn, patchString st return err } + existingSchemaByName := map[string]client.SchemaDescription{} + for _, col := range collectionsByName { + existingSchemaByName[col.Schema.Name] = col.Schema + } + // Here we swap out any string representations of enums for their integer values patch, err = substituteSchemaPatch(patch, collectionsByName) if err != nil { @@ -138,21 +136,33 @@ func (db *db) patchSchema(ctx context.Context, txn datastore.Txn, patchString st return err } - newDescriptions := []client.CollectionDescription{} + newCollections := []client.CollectionDefinition{} + newSchemaByName := map[string]client.SchemaDescription{} for _, desc := range newDescriptionsByName { - newDescriptions = append(newDescriptions, desc) + def := client.CollectionDefinition{Description: desc, Schema: desc.Schema} + + newCollections = append(newCollections, def) + newSchemaByName[def.Schema.Name] = def.Schema } - for i, desc := range newDescriptions { - col, err := db.updateCollection(ctx, txn, collectionsByName, newDescriptionsByName, desc, setAsDefaultVersion) + for i, col := range newCollections { + col, err := db.updateCollection( + ctx, + txn, + collectionsByName, + existingSchemaByName, + newSchemaByName, + col, + setAsDefaultVersion, + ) if err != nil { return err } - newDescriptions[i] = col.Description() + newCollections[i] = col.Definition() } - return db.parser.SetSchema(ctx, txn, newDescriptions) + return db.parser.SetSchema(ctx, txn, newCollections) } func (db *db) getCollectionsByName( diff --git a/http/client.go b/http/client.go index 79ff9e559b..21006f2194 100644 --- a/http/client.go +++ b/http/client.go @@ -267,11 +267,11 @@ func (c *Client) GetCollectionByName(ctx context.Context, name client.Collection if err != nil { return nil, err } - var description client.CollectionDescription - if err := c.http.requestJson(req, &description); err != nil { + var definition client.CollectionDefinition + if err := c.http.requestJson(req, &definition); err != nil { return nil, err } - return &Collection{c.http, description}, nil + return &Collection{c.http, definition}, nil } func (c *Client) GetCollectionBySchemaID(ctx context.Context, schemaId string) (client.Collection, error) { @@ -282,11 +282,11 @@ func (c *Client) GetCollectionBySchemaID(ctx context.Context, schemaId string) ( if err != nil { return nil, err } - var description client.CollectionDescription - if err := c.http.requestJson(req, &description); err != nil { + var definition client.CollectionDefinition + if err := c.http.requestJson(req, &definition); err != nil { return nil, err } - return &Collection{c.http, description}, nil + return &Collection{c.http, definition}, nil } func (c *Client) GetCollectionByVersionID(ctx context.Context, versionId string) (client.Collection, error) { @@ -297,11 +297,11 @@ func (c *Client) GetCollectionByVersionID(ctx context.Context, versionId string) if err != nil { return nil, err } - var description client.CollectionDescription - if err := c.http.requestJson(req, &description); err != nil { + var definition client.CollectionDefinition + if err := c.http.requestJson(req, &definition); err != nil { return nil, err } - return &Collection{c.http, description}, nil + return &Collection{c.http, definition}, nil } func (c *Client) GetAllCollections(ctx context.Context) ([]client.Collection, error) { @@ -311,7 +311,7 @@ func (c *Client) GetAllCollections(ctx context.Context) ([]client.Collection, er if err != nil { return nil, err } - var descriptions []client.CollectionDescription + var descriptions []client.CollectionDefinition if err := c.http.requestJson(req, &descriptions); err != nil { return nil, err } diff --git a/http/client_collection.go b/http/client_collection.go index 9641157d1b..1bb1e9e29e 100644 --- a/http/client_collection.go +++ b/http/client_collection.go @@ -32,35 +32,39 @@ var _ client.Collection = (*Collection)(nil) // Collection implements the client.Collection interface over HTTP. type Collection struct { http *httpClient - desc client.CollectionDescription + def client.CollectionDefinition } func (c *Collection) Description() client.CollectionDescription { - return c.desc + return c.def.Description } func (c *Collection) Name() string { - return c.desc.Name + return c.Description().Name } func (c *Collection) Schema() client.SchemaDescription { - return c.desc.Schema + return c.def.Schema } func (c *Collection) ID() uint32 { - return c.desc.ID + return c.Description().ID } func (c *Collection) SchemaID() string { - return c.desc.Schema.SchemaID + return c.Schema().SchemaID +} + +func (c *Collection) Definition() client.CollectionDefinition { + return c.def } func (c *Collection) Create(ctx context.Context, doc *client.Document) error { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) // We must call this here, else the doc key on the given object will not match // that of the document saved in the database - err := doc.RemapAliasFieldsAndDockey(c.Description().Schema.Fields) + err := doc.RemapAliasFieldsAndDockey(c.Schema().Fields) if err != nil { return err } @@ -82,13 +86,13 @@ func (c *Collection) Create(ctx context.Context, doc *client.Document) error { } func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) error { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) var docMapList []json.RawMessage for _, doc := range docs { // We must call this here, else the doc key on the given object will not match // that of the document saved in the database - err := doc.RemapAliasFieldsAndDockey(c.Description().Schema.Fields) + err := doc.RemapAliasFieldsAndDockey(c.Schema().Fields) if err != nil { return err } @@ -118,7 +122,7 @@ func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) er } func (c *Collection) Update(ctx context.Context, doc *client.Document) error { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name, doc.Key().String()) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, doc.Key().String()) body, err := doc.ToJSONPatch() if err != nil { @@ -148,7 +152,7 @@ func (c *Collection) Save(ctx context.Context, doc *client.Document) error { } func (c *Collection) Delete(ctx context.Context, docKey client.DocKey) (bool, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name, docKey.String()) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, docKey.String()) req, err := http.NewRequestWithContext(ctx, http.MethodDelete, methodURL.String(), nil) if err != nil { @@ -186,7 +190,7 @@ func (c *Collection) updateWith( ctx context.Context, request CollectionUpdateRequest, ) (*client.UpdateResult, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) body, err := json.Marshal(request) if err != nil { @@ -257,7 +261,7 @@ func (c *Collection) deleteWith( ctx context.Context, request CollectionDeleteRequest, ) (*client.DeleteResult, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) body, err := json.Marshal(request) if err != nil { @@ -302,7 +306,7 @@ func (c *Collection) Get(ctx context.Context, key client.DocKey, showDeleted boo query.Add("show_deleted", "true") } - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name, key.String()) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, key.String()) methodURL.RawQuery = query.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, methodURL.String(), nil) @@ -324,12 +328,12 @@ func (c *Collection) Get(ctx context.Context, key client.DocKey, showDeleted boo func (c *Collection) WithTxn(tx datastore.Txn) client.Collection { return &Collection{ http: c.http.withTxn(tx.ID()), - desc: c.desc, + def: c.def, } } func (c *Collection) GetAllDocKeys(ctx context.Context) (<-chan client.DocKeysResult, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) req, err := http.NewRequestWithContext(ctx, http.MethodGet, methodURL.String(), nil) if err != nil { @@ -381,7 +385,7 @@ func (c *Collection) CreateIndex( ctx context.Context, indexDesc client.IndexDescription, ) (client.IndexDescription, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name, "indexes") + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, "indexes") body, err := json.Marshal(&indexDesc) if err != nil { @@ -399,7 +403,7 @@ func (c *Collection) CreateIndex( } func (c *Collection) DropIndex(ctx context.Context, indexName string) error { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name, "indexes", indexName) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, "indexes", indexName) req, err := http.NewRequestWithContext(ctx, http.MethodDelete, methodURL.String(), nil) if err != nil { @@ -410,7 +414,7 @@ func (c *Collection) DropIndex(ctx context.Context, indexName string) error { } func (c *Collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.desc.Name, "indexes") + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, "indexes") req, err := http.NewRequestWithContext(ctx, http.MethodGet, methodURL.String(), nil) if err != nil { @@ -420,5 +424,5 @@ func (c *Collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, if err := c.http.requestJson(req, &indexes); err != nil { return nil, err } - return c.desc.Indexes, nil + return c.Description().Indexes, nil } diff --git a/http/handler_store.go b/http/handler_store.go index 93563c2f90..6361a7b900 100644 --- a/http/handler_store.go +++ b/http/handler_store.go @@ -200,30 +200,30 @@ func (s *storeHandler) GetCollection(rw http.ResponseWriter, req *http.Request) responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return } - responseJSON(rw, http.StatusOK, col.Description()) + responseJSON(rw, http.StatusOK, col.Definition()) case req.URL.Query().Has("schema_id"): col, err := store.GetCollectionBySchemaID(req.Context(), req.URL.Query().Get("schema_id")) if err != nil { responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return } - responseJSON(rw, http.StatusOK, col.Description()) + responseJSON(rw, http.StatusOK, col.Definition()) case req.URL.Query().Has("version_id"): col, err := store.GetCollectionByVersionID(req.Context(), req.URL.Query().Get("version_id")) if err != nil { responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return } - responseJSON(rw, http.StatusOK, col.Description()) + responseJSON(rw, http.StatusOK, col.Definition()) default: cols, err := store.GetAllCollections(req.Context()) if err != nil { responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return } - colDesc := make([]client.CollectionDescription, len(cols)) + colDesc := make([]client.CollectionDefinition, len(cols)) for i, col := range cols { - colDesc[i] = col.Description() + colDesc[i] = col.Definition() } responseJSON(rw, http.StatusOK, colDesc) } diff --git a/lens/fetcher.go b/lens/fetcher.go index ee01aa7983..23adc8671d 100644 --- a/lens/fetcher.go +++ b/lens/fetcher.go @@ -34,7 +34,7 @@ type lensedFetcher struct { txn datastore.Txn - col *client.CollectionDescription + col client.Collection // Cache the fieldDescriptions mapped by name to allow for cheaper access within the fetcher loop fieldDescriptionsByName map[string]client.FieldDescription @@ -58,7 +58,7 @@ func NewFetcher(source fetcher.Fetcher, registry client.LensRegistry) fetcher.Fe func (f *lensedFetcher) Init( ctx context.Context, txn datastore.Txn, - col *client.CollectionDescription, + col client.Collection, fields []client.FieldDescription, filter *mapper.Filter, docmapper *core.DocumentMapping, @@ -67,12 +67,12 @@ func (f *lensedFetcher) Init( ) error { f.col = col - f.fieldDescriptionsByName = make(map[string]client.FieldDescription, len(col.Schema.Fields)) + f.fieldDescriptionsByName = make(map[string]client.FieldDescription, len(col.Schema().Fields)) // Add cache the field descriptions in reverse, allowing smaller-index fields to overwrite any later // ones. This should never really happen here, but it ensures the result is consistent with col.GetField // which returns the first one it finds with a matching name. - for i := len(col.Schema.Fields) - 1; i >= 0; i-- { - field := col.Schema.Fields[i] + for i := len(col.Schema().Fields) - 1; i >= 0; i-- { + field := col.Schema().Fields[i] f.fieldDescriptionsByName[field.Name] = field } @@ -81,11 +81,11 @@ func (f *lensedFetcher) Init( return err } - history, err := getTargetedSchemaHistory(ctx, txn, cfg, f.col.Schema.SchemaID, f.col.Schema.VersionID) + history, err := getTargetedSchemaHistory(ctx, txn, cfg, f.col.Schema().SchemaID, f.col.Schema().VersionID) if err != nil { return err } - f.lens = new(ctx, f.registry, f.col.Schema.VersionID, history) + f.lens = new(ctx, f.registry, f.col.Schema().VersionID, history) f.txn = txn for schemaVersionID := range history { @@ -100,7 +100,7 @@ func (f *lensedFetcher) Init( } } - f.targetVersionID = col.Schema.VersionID + f.targetVersionID = col.Schema().VersionID var innerFetcherFields []client.FieldDescription if f.hasMigrations { @@ -238,7 +238,7 @@ func (f *lensedFetcher) lensDocToEncodedDoc(docAsMap LensDoc) (fetcher.EncodedDo return &lensEncodedDocument{ key: []byte(key), - schemaVersionID: f.col.Schema.VersionID, + schemaVersionID: f.col.Schema().VersionID, status: status, properties: properties, }, nil @@ -283,7 +283,7 @@ func (f *lensedFetcher) updateDataStore(ctx context.Context, original map[string } datastoreKeyBase := core.DataStoreKey{ - CollectionID: f.col.IDString(), + CollectionID: f.col.Description().IDString(), DocKey: dockey, InstanceType: core.ValueKey, } diff --git a/net/process.go b/net/process.go index a2fd446cfe..c07800b51f 100644 --- a/net/process.go +++ b/net/process.go @@ -102,7 +102,7 @@ func initCRDTForType( core.COMPOSITE_NAMESPACE, ) } else { - fd, ok := description.Schema.GetField(field) + fd, ok := col.Schema().GetField(field) if !ok { return nil, errors.New(fmt.Sprintf("Couldn't find field %s for doc %s", field, dsKey)) } diff --git a/planner/commit.go b/planner/commit.go index e6216e2b43..c2cff28c30 100644 --- a/planner/commit.go +++ b/planner/commit.go @@ -333,7 +333,7 @@ func (n *dagScanNode) dagBlockToNodeDoc(block blocks.Block) (core.Doc, []*ipld.L return core.Doc{}, nil, err } - field, ok := c.Description().Schema.GetField(fieldName.(string)) + field, ok := c.Schema().GetField(fieldName.(string)) if !ok { return core.Doc{}, nil, client.NewErrFieldNotExist(fieldName.(string)) } diff --git a/planner/datasource.go b/planner/datasource.go index 862f43bd33..72ac7579b4 100644 --- a/planner/datasource.go +++ b/planner/datasource.go @@ -15,15 +15,9 @@ import ( "github.com/sourcenetwork/defradb/planner/mapper" ) -// sourceInfo stores info about the data source -type sourceInfo struct { - collectionDescription client.CollectionDescription - // and more -} - type planSource struct { - info sourceInfo - plan planNode + collection client.Collection + plan planNode } func (p *Planner) getSource(parsed *mapper.Select) (planSource, error) { @@ -43,9 +37,7 @@ func (p *Planner) getCollectionScanPlan(mapperSelect *mapper.Select) (planSource } return planSource{ - plan: scan, - info: sourceInfo{ - collectionDescription: col.Description(), - }, + plan: scan, + collection: col, }, nil } diff --git a/planner/mapper/descriptions.go b/planner/mapper/descriptions.go deleted file mode 100644 index e7edd865cd..0000000000 --- a/planner/mapper/descriptions.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package mapper - -import ( - "context" - "encoding/json" - - "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/core" - "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/errors" -) - -// DescriptionsRepo is a cache of previously requested collection descriptions -// that can be used to reduce multiple reads of the same collection description. -type DescriptionsRepo struct { - ctx context.Context - txn datastore.Txn - - collectionDescriptionsByName map[string]client.CollectionDescription -} - -// NewDescriptionsRepo instantiates a new DescriptionsRepo with the given context and transaction. -func NewDescriptionsRepo(ctx context.Context, txn datastore.Txn) *DescriptionsRepo { - return &DescriptionsRepo{ - ctx: ctx, - txn: txn, - collectionDescriptionsByName: map[string]client.CollectionDescription{}, - } -} - -// getCollectionDesc returns the description of the collection with the given name. -// -// Will return nil and an error if a description of the given name is not found. Will first look -// in the repo's cache for the description before doing a query operation on the datastore. -func (r *DescriptionsRepo) getCollectionDesc(name string) (client.CollectionDescription, error) { - collectionKey := core.NewCollectionKey(name) - var desc client.CollectionDescription - schemaVersionIdBytes, err := r.txn.Systemstore().Get(r.ctx, collectionKey.ToDS()) - if err != nil { - return desc, errors.Wrap("failed to get collection description", err) - } - - schemaVersionId := string(schemaVersionIdBytes) - schemaVersionKey := core.NewCollectionSchemaVersionKey(schemaVersionId) - buf, err := r.txn.Systemstore().Get(r.ctx, schemaVersionKey.ToDS()) - if err != nil { - return desc, err - } - - err = json.Unmarshal(buf, &desc) - if err != nil { - return desc, err - } - - return desc, nil -} diff --git a/planner/mapper/mapper.go b/planner/mapper/mapper.go index b6f80a55a2..418c0c5c57 100644 --- a/planner/mapper/mapper.go +++ b/planner/mapper/mapper.go @@ -21,7 +21,6 @@ import ( "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/connor" "github.com/sourcenetwork/defradb/core" - "github.com/sourcenetwork/defradb/datastore" ) var ( @@ -32,10 +31,9 @@ var ( // // In the process of doing so it will construct the document map required to access the data // yielded by the [Select]. -func ToSelect(ctx context.Context, txn datastore.Txn, selectRequest *request.Select) (*Select, error) { - descriptionsRepo := NewDescriptionsRepo(ctx, txn) +func ToSelect(ctx context.Context, store client.Store, selectRequest *request.Select) (*Select, error) { // the top-level select will always have index=0, and no parent collection name - return toSelect(descriptionsRepo, 0, selectRequest, "") + return toSelect(ctx, store, 0, selectRequest, "") } // toSelect converts the given [parser.Select] into a [Select]. @@ -43,29 +41,30 @@ func ToSelect(ctx context.Context, txn datastore.Txn, selectRequest *request.Sel // In the process of doing so it will construct the document map required to access the data // yielded by the [Select]. func toSelect( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, thisIndex int, selectRequest *request.Select, parentCollectionName string, ) (*Select, error) { - collectionName, err := getCollectionName(descriptionsRepo, selectRequest, parentCollectionName) + collectionName, err := getCollectionName(ctx, store, selectRequest, parentCollectionName) if err != nil { return nil, err } - mapping, desc, err := getTopLevelInfo(descriptionsRepo, selectRequest, collectionName) + mapping, collection, err := getTopLevelInfo(ctx, store, selectRequest, collectionName) if err != nil { return nil, err } - fields, aggregates, err := getRequestables(selectRequest, mapping, desc, descriptionsRepo) + fields, aggregates, err := getRequestables(ctx, selectRequest, mapping, collection, store) if err != nil { return nil, err } // Needs to be done before resolving aggregates, else filter conversion may fail there filterDependencies, err := resolveFilterDependencies( - descriptionsRepo, collectionName, selectRequest.Filter, mapping, fields) + ctx, store, collectionName, selectRequest.Filter, mapping, fields) if err != nil { return nil, err } @@ -73,28 +72,31 @@ func toSelect( // Resolve order dependencies that may have been missed due to not being rendered. err = resolveOrderDependencies( - descriptionsRepo, collectionName, selectRequest.OrderBy, mapping, &fields) + ctx, store, collectionName, selectRequest.OrderBy, mapping, &fields) if err != nil { return nil, err } aggregates = appendUnderlyingAggregates(aggregates, mapping) fields, err = resolveAggregates( + ctx, selectRequest, aggregates, fields, mapping, - desc, - descriptionsRepo, + collection, + store, ) if err != nil { return nil, err } - fields, err = resolveSecondaryRelationIDs(descriptionsRepo, desc, mapping, fields) - if err != nil { - return nil, err + if collection != nil { + fields, err = resolveSecondaryRelationIDs(ctx, store, collection, mapping, fields) + if err != nil { + return nil, err + } } // Resolve groupBy mappings i.e. alias remapping and handle missed inner group. @@ -102,7 +104,10 @@ func toSelect( groupByFields := selectRequest.GroupBy.Value().Fields // Remap all alias field names to use their internal field name mappings. for index, groupByField := range groupByFields { - fieldDesc, ok := desc.Schema.GetField(groupByField) + if collection == nil { + continue + } + fieldDesc, ok := collection.Schema().GetField(groupByField) if ok && fieldDesc.IsObject() && !fieldDesc.IsObjectArray() { groupByFields[index] = groupByField + request.RelatedObjectID } else if ok && fieldDesc.IsObjectArray() { @@ -135,7 +140,8 @@ func toSelect( // resolveOrderDependencies will map fields that were missed due to them not being requested. // Modifies the consumed existingFields and mapping accordingly. func resolveOrderDependencies( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, descName string, source immutable.Option[request.OrderBy], mapping *core.DocumentMapping, @@ -160,7 +166,7 @@ outer: joinField := fields[0] // ensure the child select is resolved for this order join - innerSelect, err := resolveChildOrder(descriptionsRepo, descName, joinField, mapping, currentExistingFields) + innerSelect, err := resolveChildOrder(ctx, store, descName, joinField, mapping, currentExistingFields) if err != nil { return err } @@ -178,7 +184,7 @@ outer: joinField := fields[0] // ensure the child select is resolved for this order join - innerSelect, err := resolveChildOrder(descriptionsRepo, descName, joinField, mapping, existingFields) + innerSelect, err := resolveChildOrder(ctx, store, descName, joinField, mapping, existingFields) if err != nil { return err } @@ -203,7 +209,8 @@ outer: // given a type join field, ensure its mapping exists // and add a coorsponding select field(s) func resolveChildOrder( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, descName string, orderChildField string, mapping *core.DocumentMapping, @@ -221,7 +228,7 @@ func resolveChildOrder( Name: orderChildField, }, } - innerSelect, err := toSelect(descriptionsRepo, index, &dummyJoinFieldSelect, descName) + innerSelect, err := toSelect(ctx, store, index, &dummyJoinFieldSelect, descName) if err != nil { return nil, err } @@ -250,12 +257,13 @@ func resolveChildOrder( // append the new target field as well as the aggregate. The mapping will also be // updated with any new fields/aggregates. func resolveAggregates( + ctx context.Context, selectRequest *request.Select, aggregates []*aggregateRequest, inputFields []Requestable, mapping *core.DocumentMapping, - desc *client.CollectionDescription, - descriptionsRepo *DescriptionsRepo, + collection client.Collection, + store client.Store, ) ([]Requestable, error) { fields := inputFields dependenciesByParentId := map[int][]int{} @@ -274,7 +282,12 @@ func resolveAggregates( var hasHost bool var convertedFilter *Filter if childIsMapped { - fieldDesc, isField := desc.Schema.GetField(target.hostExternalName) + var fieldDesc client.FieldDescription + var isField bool + if collection != nil { + fieldDesc, isField = collection.Schema().GetField(target.hostExternalName) + } + if isField && !fieldDesc.IsObject() { var order *OrderBy if target.order.HasValue() && len(target.order.Value().Conditions) > 0 { @@ -326,24 +339,29 @@ func resolveAggregates( }, } - childCollectionName, err := getCollectionName(descriptionsRepo, hostSelectRequest, desc.Name) + var collectionName string + if collection != nil { + collectionName = collection.Name() + } + + childCollectionName, err := getCollectionName(ctx, store, hostSelectRequest, collectionName) if err != nil { return nil, err } mapAggregateNestedTargets(target, hostSelectRequest, selectRequest.Root) - childMapping, childDesc, err := getTopLevelInfo(descriptionsRepo, hostSelectRequest, childCollectionName) + childMapping, childDesc, err := getTopLevelInfo(ctx, store, hostSelectRequest, childCollectionName) if err != nil { return nil, err } - childFields, _, err := getRequestables(hostSelectRequest, childMapping, childDesc, descriptionsRepo) + childFields, _, err := getRequestables(ctx, hostSelectRequest, childMapping, childDesc, store) if err != nil { return nil, err } err = resolveOrderDependencies( - descriptionsRepo, childCollectionName, target.order, childMapping, &childFields) + ctx, store, childCollectionName, target.order, childMapping, &childFields) if err != nil { return nil, err } @@ -587,10 +605,11 @@ func appendIfNotExists( // and aggregateRequests from the given selectRequest.Fields slice. It also mutates the // consumed mapping data. func getRequestables( + ctx context.Context, selectRequest *request.Select, mapping *core.DocumentMapping, - desc *client.CollectionDescription, - descriptionsRepo *DescriptionsRepo, + collection client.Collection, + store client.Store, ) (fields []Requestable, aggregates []*aggregateRequest, err error) { for _, field := range selectRequest.Fields { switch f := field.(type) { @@ -611,8 +630,12 @@ func getRequestables( }) case *request.Select: index := mapping.GetNextIndex() + var parentCollectionName string + if collection != nil { + parentCollectionName = collection.Name() + } - innerSelect, err := toSelect(descriptionsRepo, index, f, desc.Name) + innerSelect, err := toSelect(ctx, store, index, f, parentCollectionName) if err != nil { return nil, nil, err } @@ -676,7 +699,8 @@ func getAggregateRequests(index int, aggregate *request.Aggregate) (aggregateReq // getCollectionName returns the name of the selectRequest collection. This may be empty // if this is a commit request. func getCollectionName( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, selectRequest *request.Select, parentCollectionName string, ) (string, error) { @@ -692,12 +716,12 @@ func getCollectionName( } if parentCollectionName != "" { - parentDescription, err := descriptionsRepo.getCollectionDesc(parentCollectionName) + parentCollection, err := store.GetCollectionByName(ctx, parentCollectionName) if err != nil { return "", err } - hostFieldDesc, parentHasField := parentDescription.Schema.GetField(selectRequest.Name) + hostFieldDesc, parentHasField := parentCollection.Schema().GetField(selectRequest.Name) if parentHasField && hostFieldDesc.RelationType != 0 { // If this field exists on the parent, and it is a child object // then this collection name is the collection name of the child. @@ -710,28 +734,29 @@ func getCollectionName( // getTopLevelInfo returns the collection description and maps the fields directly on the object. func getTopLevelInfo( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, selectRequest *request.Select, collectionName string, -) (*core.DocumentMapping, *client.CollectionDescription, error) { +) (*core.DocumentMapping, client.Collection, error) { mapping := core.NewDocumentMapping() if _, isAggregate := request.Aggregates[selectRequest.Name]; isAggregate { // If this is a (top-level) aggregate, then it will have no collection // description, and no top-level fields, so we return an empty mapping only - return mapping, &client.CollectionDescription{}, nil + return mapping, nil, nil } if selectRequest.Root == request.ObjectSelection { mapping.Add(core.DocKeyFieldIndex, request.KeyFieldName) - desc, err := descriptionsRepo.getCollectionDesc(collectionName) + collection, err := store.GetCollectionByName(ctx, collectionName) if err != nil { return nil, nil, err } // Map all fields from schema into the map as they are fetched automatically - for _, f := range desc.Schema.Fields { + for _, f := range collection.Schema().Fields { if f.IsObject() { // Objects are skipped, as they are not fetched by default and // have to be requested via selects. @@ -746,7 +771,7 @@ func getTopLevelInfo( mapping.Add(mapping.GetNextIndex(), request.DeletedFieldName) - return mapping, &desc, nil + return mapping, collection, nil } if selectRequest.Name == request.LinksFieldName { @@ -767,11 +792,12 @@ func getTopLevelInfo( mapping.SetTypeName(request.CommitTypeName) } - return mapping, &client.CollectionDescription{}, nil + return mapping, nil, nil } func resolveFilterDependencies( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, parentCollectionName string, source immutable.Option[request.Filter], mapping *core.DocumentMapping, @@ -782,7 +808,8 @@ func resolveFilterDependencies( } return resolveInnerFilterDependencies( - descriptionsRepo, + ctx, + store, parentCollectionName, source.Value().Conditions, mapping, @@ -792,7 +819,8 @@ func resolveFilterDependencies( } func resolveInnerFilterDependencies( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, parentCollectionName string, source map[string]any, mapping *core.DocumentMapping, @@ -806,7 +834,8 @@ func resolveInnerFilterDependencies( compoundFilter := source[key].([]any) for _, innerFilter := range compoundFilter { innerFields, err := resolveInnerFilterDependencies( - descriptionsRepo, + ctx, + store, parentCollectionName, innerFilter.(map[string]any), mapping, @@ -824,7 +853,8 @@ func resolveInnerFilterDependencies( } else if key == request.FilterOpNot { notFilter := source[key].(map[string]any) innerFields, err := resolveInnerFilterDependencies( - descriptionsRepo, + ctx, + store, parentCollectionName, notFilter, mapping, @@ -868,7 +898,7 @@ func resolveInnerFilterDependencies( } } else { var err error - childSelect, err = constructEmptyJoin(descriptionsRepo, parentCollectionName, mapping, key) + childSelect, err = constructEmptyJoin(ctx, store, parentCollectionName, mapping, key) if err != nil { return nil, err } @@ -885,13 +915,14 @@ func resolveInnerFilterDependencies( } dummyParsed := &request.Select{Field: request.Field{Name: key}} - childCollectionName, err := getCollectionName(descriptionsRepo, dummyParsed, parentCollectionName) + childCollectionName, err := getCollectionName(ctx, store, dummyParsed, parentCollectionName) if err != nil { return nil, err } childFields, err := resolveInnerFilterDependencies( - descriptionsRepo, + ctx, + store, childCollectionName, childFilter, childSelect.DocumentMapping, @@ -910,7 +941,8 @@ func resolveInnerFilterDependencies( // constructEmptyJoin constructs a valid empty join with no requested fields. func constructEmptyJoin( - descriptionsRepo *DescriptionsRepo, + ctx context.Context, + store client.Store, parentCollectionName string, parentMapping *core.DocumentMapping, name string, @@ -923,12 +955,12 @@ func constructEmptyJoin( }, } - childCollectionName, err := getCollectionName(descriptionsRepo, dummyParsed, parentCollectionName) + childCollectionName, err := getCollectionName(ctx, store, dummyParsed, parentCollectionName) if err != nil { return nil, err } - childMapping, _, err := getTopLevelInfo(descriptionsRepo, dummyParsed, childCollectionName) + childMapping, _, err := getTopLevelInfo(ctx, store, dummyParsed, childCollectionName) if err != nil { return nil, err } @@ -955,8 +987,9 @@ func constructEmptyJoin( // // They copying itself is handled within [typeJoinOne]. func resolveSecondaryRelationIDs( - descriptionsRepo *DescriptionsRepo, - desc *client.CollectionDescription, + ctx context.Context, + store client.Store, + collection client.Collection, mapping *core.DocumentMapping, requestables []Requestable, ) ([]Requestable, error) { @@ -968,7 +1001,7 @@ func resolveSecondaryRelationIDs( continue } - fieldDesc, descFound := desc.Schema.GetField(existingField.Name) + fieldDesc, descFound := collection.Schema().GetField(existingField.Name) if !descFound { continue } @@ -977,7 +1010,7 @@ func resolveSecondaryRelationIDs( continue } - objectFieldDesc, descFound := desc.Schema.GetField( + objectFieldDesc, descFound := collection.Schema().GetField( strings.TrimSuffix(existingField.Name, request.RelatedObjectID), ) if !descFound { @@ -995,7 +1028,7 @@ func resolveSecondaryRelationIDs( continue } - siblingFieldDesc, descFound := desc.Schema.GetField(siblingSelect.Field.Name) + siblingFieldDesc, descFound := collection.Schema().GetField(siblingSelect.Field.Name) if !descFound { continue } @@ -1017,8 +1050,9 @@ func resolveSecondaryRelationIDs( // We only require the dockey of the related object, so an empty join is all we need. join, err := constructEmptyJoin( - descriptionsRepo, - desc.Name, + ctx, + store, + collection.Name(), mapping, objectFieldName, ) @@ -1039,10 +1073,10 @@ func resolveSecondaryRelationIDs( // yielded by the [Select] embedded in the [CommitSelect]. func ToCommitSelect( ctx context.Context, - txn datastore.Txn, + store client.Store, selectRequest *request.CommitSelect, ) (*CommitSelect, error) { - underlyingSelect, err := ToSelect(ctx, txn, selectRequest.ToSelect()) + underlyingSelect, err := ToSelect(ctx, store, selectRequest.ToSelect()) if err != nil { return nil, err } @@ -1059,8 +1093,8 @@ func ToCommitSelect( // // In the process of doing so it will construct the document map required to access the data // yielded by the [Select] embedded in the [Mutation]. -func ToMutation(ctx context.Context, txn datastore.Txn, mutationRequest *request.ObjectMutation) (*Mutation, error) { - underlyingSelect, err := ToSelect(ctx, txn, mutationRequest.ToSelect()) +func ToMutation(ctx context.Context, store client.Store, mutationRequest *request.ObjectMutation) (*Mutation, error) { + underlyingSelect, err := ToSelect(ctx, store, mutationRequest.ToSelect()) if err != nil { return nil, err } diff --git a/planner/planner.go b/planner/planner.go index 7821b5aaaf..b066e1f0e3 100644 --- a/planner/planner.go +++ b/planner/planner.go @@ -114,7 +114,7 @@ func (p *Planner) newPlan(stmt any) (planNode, error) { return p.newPlan(n.Selections[0]) case *request.Select: - m, err := mapper.ToSelect(p.ctx, p.txn, n) + m, err := mapper.ToSelect(p.ctx, p.db, n) if err != nil { return nil, err } @@ -129,14 +129,14 @@ func (p *Planner) newPlan(stmt any) (planNode, error) { return p.Select(m) case *request.CommitSelect: - m, err := mapper.ToCommitSelect(p.ctx, p.txn, n) + m, err := mapper.ToCommitSelect(p.ctx, p.db, n) if err != nil { return nil, err } return p.CommitSelect(m) case *request.ObjectMutation: - m, err := mapper.ToMutation(p.ctx, p.txn, n) + m, err := mapper.ToMutation(p.ctx, p.db, n) if err != nil { return nil, err } @@ -338,8 +338,9 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan node.documentMapping, ) slct := node.subType.(*selectTopNode).selectNode - desc := slct.sourceInfo.collectionDescription - indexedFields := desc.CollectIndexedFields(&desc.Schema) + desc := slct.collection.Description() + schema := slct.collection.Schema() + indexedFields := desc.CollectIndexedFields(&schema) for _, indField := range indexedFields { if ind, ok := filteredSubFields[indField.Name]; ok { subInd := node.documentMapping.FirstIndexOfName(node.subTypeName) @@ -412,7 +413,7 @@ func (p *Planner) expandGroupNodePlan(topNodeSelect *selectTopNode) error { childSelect, pipe, false, - &topNodeSelect.selectNode.sourceInfo, + topNodeSelect.selectNode.collection, ) if err != nil { return err diff --git a/planner/scan.go b/planner/scan.go index f9a80705cb..64a534da6d 100644 --- a/planner/scan.go +++ b/planner/scan.go @@ -38,8 +38,8 @@ type scanNode struct { documentIterator docMapper - p *Planner - desc client.CollectionDescription + p *Planner + col client.Collection fields []client.FieldDescription @@ -65,7 +65,7 @@ func (n *scanNode) Init() error { if err := n.fetcher.Init( n.p.ctx, n.p.txn, - &n.desc, + n.col, n.fields, n.filter, n.slct.DocumentMapping, @@ -77,8 +77,8 @@ func (n *scanNode) Init() error { return n.initScan() } -func (n *scanNode) initCollection(desc client.CollectionDescription) error { - n.desc = desc +func (n *scanNode) initCollection(col client.Collection) error { + n.col = col return n.initFields(n.slct.Fields) } @@ -104,7 +104,7 @@ func (n *scanNode) initFields(fields []mapper.Requestable) error { if target.Filter != nil { fieldDescs, err := parser.ParseFilterFieldsForDescription( target.Filter.ExternalConditions, - n.desc.Schema, + n.col.Schema(), ) if err != nil { return err @@ -125,7 +125,7 @@ func (n *scanNode) initFields(fields []mapper.Requestable) error { } func (n *scanNode) tryAddField(fieldName string) bool { - fd, ok := n.desc.Schema.GetField(fieldName) + fd, ok := n.col.Schema().GetField(fieldName) if !ok { // skip fields that are not part of the // schema description. The scanner (and fetcher) @@ -152,7 +152,7 @@ func (scan *scanNode) initFetcher( var indexFilter *mapper.Filter scan.filter, indexFilter = filter.SplitByField(scan.filter, field) if indexFilter != nil { - fieldDesc, _ := scan.desc.Schema.GetField(indexedField.Value().Name) + fieldDesc, _ := scan.col.Schema().GetField(indexedField.Value().Name) f = fetcher.NewIndexFetcher(f, fieldDesc, indexFilter) } } @@ -170,7 +170,7 @@ func (n *scanNode) Start() error { func (n *scanNode) initScan() error { if !n.spans.HasValue { - start := base.MakeCollectionKey(n.desc) + start := base.MakeCollectionKey(n.col.Description()) n.spans = core.NewSpans(core.NewSpan(start, start.PrefixEnd())) } @@ -252,8 +252,8 @@ func (n *scanNode) simpleExplain() (map[string]any, error) { } // Add the collection attributes. - simpleExplainMap[collectionNameLabel] = n.desc.Name - simpleExplainMap[collectionIDLabel] = n.desc.IDString() + simpleExplainMap[collectionNameLabel] = n.col.Name() + simpleExplainMap[collectionIDLabel] = n.col.Description().IDString() // Add the spans attribute. simpleExplainMap[spansLabel] = n.explainSpans() @@ -298,7 +298,11 @@ func (p *Planner) Scan( docMapper: docMapper{mapperSelect.DocumentMapping}, } - err := scan.initCollection(colDesc) + col, err := p.db.GetCollectionByName(p.ctx, mapperSelect.CollectionName) + if err != nil { + return nil, err + } + err = scan.initCollection(col) if err != nil { return nil, err } diff --git a/planner/select.go b/planner/select.go index 21524ed31f..20c0dd43ba 100644 --- a/planner/select.go +++ b/planner/select.go @@ -102,9 +102,7 @@ type selectNode struct { // was created origSource planNode - // cache information about the original data source - // collection name, meta-data, etc. - sourceInfo sourceInfo + collection client.Collection // top level filter expression // filter is split between select, scan, and typeIndexJoin. @@ -245,7 +243,7 @@ func (n *selectNode) initSource() ([]aggregateNode, error) { } n.source = sourcePlan.plan n.origSource = sourcePlan.plan - n.sourceInfo = sourcePlan.info + n.collection = sourcePlan.collection // split filter // apply the root filter to the source @@ -279,7 +277,7 @@ func (n *selectNode) initSource() ([]aggregateNode, error) { // instead of a prefix scan + filter via the Primary Index (0), like here: spans := make([]core.Span, len(n.selectReq.DocKeys.Value())) for i, docKey := range n.selectReq.DocKeys.Value() { - dockeyIndexKey := base.MakeDocKey(sourcePlan.info.collectionDescription, docKey) + dockeyIndexKey := base.MakeDocKey(sourcePlan.collection.Description(), docKey) spans[i] = core.NewSpan(dockeyIndexKey, dockeyIndexKey.PrefixEnd()) } origScan.Spans(core.NewSpans(spans...)) @@ -300,7 +298,8 @@ func (n *selectNode) initSource() ([]aggregateNode, error) { func findFilteredByIndexedField(scanNode *scanNode) immutable.Option[client.FieldDescription] { if scanNode.filter != nil { - indexedFields := scanNode.desc.CollectIndexedFields(&scanNode.desc.Schema) + schema := scanNode.col.Schema() + indexedFields := scanNode.col.Description().CollectIndexedFields(&schema) for i := range indexedFields { typeIndex := scanNode.documentMapping.FirstIndexOfName(indexedFields[i].Name) if scanNode.filter.HasIndex(typeIndex) { @@ -404,7 +403,7 @@ func (p *Planner) SelectFromSource( selectReq *mapper.Select, source planNode, fromCollection bool, - providedSourceInfo *sourceInfo, + collection client.Collection, ) (planNode, error) { s := &selectNode{ planner: p, @@ -419,8 +418,8 @@ func (p *Planner) SelectFromSource( orderBy := selectReq.OrderBy groupBy := selectReq.GroupBy - if providedSourceInfo != nil { - s.sourceInfo = *providedSourceInfo + if collection != nil { + s.collection = collection } if fromCollection { @@ -429,7 +428,7 @@ func (p *Planner) SelectFromSource( return nil, err } - s.sourceInfo = sourceInfo{col.Description()} + s.collection = col } aggregates, err := s.initFields(selectReq) diff --git a/planner/type_join.go b/planner/type_join.go index 6e5d9a0d49..47ba07e96b 100644 --- a/planner/type_join.go +++ b/planner/type_join.go @@ -81,8 +81,7 @@ func (p *Planner) makeTypeIndexJoin( var joinPlan planNode var err error - desc := parent.sourceInfo.collectionDescription - typeFieldDesc, ok := desc.Schema.GetField(subType.Name) + typeFieldDesc, ok := parent.collection.Schema().GetField(subType.Name) if !ok { return nil, client.NewErrFieldNotExist(subType.Name) } @@ -245,7 +244,7 @@ func (p *Planner) makeTypeJoinOne( } // get the correct sub field schema type (collection) - subTypeFieldDesc, ok := parent.sourceInfo.collectionDescription.Schema.GetField(subType.Name) + subTypeFieldDesc, ok := parent.collection.Schema().GetField(subType.Name) if !ok { return nil, client.NewErrFieldNotExist(subType.Name) } @@ -258,11 +257,13 @@ func (p *Planner) makeTypeJoinOne( if err != nil { return nil, err } + subTypeSchema := subTypeCol.Schema() subTypeField, subTypeFieldNameFound := subTypeCol.Description().GetFieldByRelation( subTypeFieldDesc.RelationName, - parent.sourceInfo.collectionDescription.Name, + parent.collection.Name(), subTypeFieldDesc.Name, + &subTypeSchema, ) if !subTypeFieldNameFound { return nil, client.NewErrFieldNotExist(subTypeFieldDesc.RelationName) @@ -382,7 +383,7 @@ func (p *Planner) makeTypeJoinMany( return nil, err } - subTypeFieldDesc, ok := parent.sourceInfo.collectionDescription.Schema.GetField(subType.Name) + subTypeFieldDesc, ok := parent.collection.Schema().GetField(subType.Name) if !ok { return nil, client.NewErrFieldNotExist(subType.Name) } @@ -391,11 +392,13 @@ func (p *Planner) makeTypeJoinMany( if err != nil { return nil, err } + subTypeSchema := subTypeCol.Schema() rootField, rootNameFound := subTypeCol.Description().GetFieldByRelation( subTypeFieldDesc.RelationName, - parent.sourceInfo.collectionDescription.Name, + parent.collection.Name(), subTypeFieldDesc.Name, + &subTypeSchema, ) if !rootNameFound { @@ -441,7 +444,7 @@ func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { if scan == nil { return false, nil } - rootDocKey := base.MakeDocKey(scan.desc, docKeyStr) + rootDocKey := base.MakeDocKey(scan.col.Description(), docKeyStr) spans := core.NewSpans(core.NewSpan(rootDocKey, rootDocKey.PrefixEnd())) diff --git a/request/graphql/parser.go b/request/graphql/parser.go index ddd13d9e62..743c3eab97 100644 --- a/request/graphql/parser.go +++ b/request/graphql/parser.go @@ -104,13 +104,13 @@ func (p *parser) Parse(ast *ast.Document) (*request.Request, []error) { } func (p *parser) ParseSDL(ctx context.Context, schemaString string) ( - []client.CollectionDescription, + []client.CollectionDefinition, error, ) { return schema.FromString(ctx, schemaString) } -func (p *parser) SetSchema(ctx context.Context, txn datastore.Txn, collections []client.CollectionDescription) error { +func (p *parser) SetSchema(ctx context.Context, txn datastore.Txn, collections []client.CollectionDefinition) error { schemaManager, err := schema.NewSchemaManager() if err != nil { return err diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 00287c4454..d5b55fb6da 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -26,7 +26,7 @@ import ( // FromString parses a GQL SDL string into a set of collection descriptions. func FromString(ctx context.Context, schemaString string) ( - []client.CollectionDescription, + []client.CollectionDefinition, error, ) { source := source.NewSource(&source.Source{ @@ -47,11 +47,11 @@ func FromString(ctx context.Context, schemaString string) ( // fromAst parses a GQL AST into a set of collection descriptions. func fromAst(ctx context.Context, doc *ast.Document) ( - []client.CollectionDescription, + []client.CollectionDefinition, error, ) { relationManager := NewRelationManager() - descriptions := []client.CollectionDescription{} + definitions := []client.CollectionDefinition{} for _, def := range doc.Definitions { switch defType := def.(type) { @@ -61,7 +61,7 @@ func fromAst(ctx context.Context, doc *ast.Document) ( return nil, err } - descriptions = append(descriptions, description) + definitions = append(definitions, description) default: // Do nothing, ignore it and continue @@ -72,12 +72,12 @@ func fromAst(ctx context.Context, doc *ast.Document) ( // The details on the relations between objects depend on both sides // of the relationship. The relation manager handles this, and must be applied // after all the collections have been processed. - err := finalizeRelations(relationManager, descriptions) + err := finalizeRelations(relationManager, definitions) if err != nil { return nil, err } - return descriptions, nil + return definitions, nil } // fromAstDefinition parses a AST object definition into a set of collection descriptions. @@ -85,7 +85,7 @@ func fromAstDefinition( ctx context.Context, relationManager *RelationManager, def *ast.ObjectDefinition, -) (client.CollectionDescription, error) { +) (client.CollectionDefinition, error) { fieldDescriptions := []client.FieldDescription{ { Name: request.KeyFieldName, @@ -98,7 +98,7 @@ func fromAstDefinition( for _, field := range def.Fields { tmpFieldsDescriptions, err := fieldsFromAST(field, relationManager, def) if err != nil { - return client.CollectionDescription{}, err + return client.CollectionDefinition{}, err } fieldDescriptions = append(fieldDescriptions, tmpFieldsDescriptions...) @@ -107,7 +107,7 @@ func fromAstDefinition( if directive.Name.Value == types.IndexDirectiveLabel { index, err := fieldIndexFromAST(field, directive) if err != nil { - return client.CollectionDescription{}, err + return client.CollectionDefinition{}, err } indexDescriptions = append(indexDescriptions, index) } @@ -129,19 +129,21 @@ func fromAstDefinition( if directive.Name.Value == types.IndexDirectiveLabel { index, err := indexFromAST(directive) if err != nil { - return client.CollectionDescription{}, err + return client.CollectionDefinition{}, err } indexDescriptions = append(indexDescriptions, index) } } - return client.CollectionDescription{ - Name: def.Name.Value, + return client.CollectionDefinition{ + Description: client.CollectionDescription{ + Name: def.Name.Value, + Indexes: indexDescriptions, + }, Schema: client.SchemaDescription{ Name: def.Name.Value, Fields: fieldDescriptions, }, - Indexes: indexDescriptions, }, nil } @@ -424,9 +426,9 @@ func getRelationshipName( return genRelationName(hostName, targetName) } -func finalizeRelations(relationManager *RelationManager, descriptions []client.CollectionDescription) error { - for _, description := range descriptions { - for i, field := range description.Schema.Fields { +func finalizeRelations(relationManager *RelationManager, definitions []client.CollectionDefinition) error { + for _, definition := range definitions { + for i, field := range definition.Schema.Fields { if field.RelationType == 0 || field.RelationType&client.Relation_Type_INTERNAL_ID != 0 { continue } @@ -447,7 +449,7 @@ func finalizeRelations(relationManager *RelationManager, descriptions []client.C } field.RelationType = rel.Kind() | fieldRelationType - description.Schema.Fields[i] = field + definition.Schema.Fields[i] = field } } diff --git a/request/graphql/schema/descriptions_test.go b/request/graphql/schema/descriptions_test.go index 2ce5e55dc9..2368b58c27 100644 --- a/request/graphql/schema/descriptions_test.go +++ b/request/graphql/schema/descriptions_test.go @@ -30,9 +30,12 @@ func TestSingleSimpleType(t *testing.T) { verified: Boolean } `, - targetDescs: []client.CollectionDescription{ + targetDescs: []client.CollectionDefinition{ { - Name: "User", + Description: client.CollectionDescription{ + Name: "User", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "User", Fields: []client.FieldDescription{ @@ -58,7 +61,6 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, }, }, @@ -77,9 +79,12 @@ func TestSingleSimpleType(t *testing.T) { rating: Float } `, - targetDescs: []client.CollectionDescription{ + targetDescs: []client.CollectionDefinition{ { - Name: "User", + Description: client.CollectionDescription{ + Name: "User", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "User", Fields: []client.FieldDescription{ @@ -105,10 +110,12 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, { - Name: "Author", + Description: client.CollectionDescription{ + Name: "Author", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Author", Fields: []client.FieldDescription{ @@ -134,7 +141,6 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, }, }, @@ -153,9 +159,12 @@ func TestSingleSimpleType(t *testing.T) { published: Book } `, - targetDescs: []client.CollectionDescription{ + targetDescs: []client.CollectionDefinition{ { - Name: "Book", + Description: client.CollectionDescription{ + Name: "Book", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Book", Fields: []client.FieldDescription{ @@ -190,10 +199,12 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, { - Name: "Author", + Description: client.CollectionDescription{ + Name: "Author", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Author", Fields: []client.FieldDescription{ @@ -228,7 +239,6 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, }, }, @@ -247,9 +257,12 @@ func TestSingleSimpleType(t *testing.T) { rating: Float } `, - targetDescs: []client.CollectionDescription{ + targetDescs: []client.CollectionDefinition{ { - Name: "User", + Description: client.CollectionDescription{ + Name: "User", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "User", Fields: []client.FieldDescription{ @@ -275,10 +288,12 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, { - Name: "Author", + Description: client.CollectionDescription{ + Name: "Author", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Author", Fields: []client.FieldDescription{ @@ -304,7 +319,6 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, }, }, @@ -323,9 +337,12 @@ func TestSingleSimpleType(t *testing.T) { published: Book @relation(name:"book_authors") } `, - targetDescs: []client.CollectionDescription{ + targetDescs: []client.CollectionDefinition{ { - Name: "Book", + Description: client.CollectionDescription{ + Name: "Book", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Book", Fields: []client.FieldDescription{ @@ -360,10 +377,12 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, { - Name: "Author", + Description: client.CollectionDescription{ + Name: "Author", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Author", Fields: []client.FieldDescription{ @@ -398,7 +417,6 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, }, }, @@ -417,9 +435,12 @@ func TestSingleSimpleType(t *testing.T) { published: Book } `, - targetDescs: []client.CollectionDescription{ + targetDescs: []client.CollectionDefinition{ { - Name: "Book", + Description: client.CollectionDescription{ + Name: "Book", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Book", Fields: []client.FieldDescription{ @@ -454,10 +475,12 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, { - Name: "Author", + Description: client.CollectionDescription{ + Name: "Author", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Author", Fields: []client.FieldDescription{ @@ -492,7 +515,6 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, }, }, @@ -511,9 +533,12 @@ func TestSingleSimpleType(t *testing.T) { published: [Book] } `, - targetDescs: []client.CollectionDescription{ + targetDescs: []client.CollectionDefinition{ { - Name: "Book", + Description: client.CollectionDescription{ + Name: "Book", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Book", Fields: []client.FieldDescription{ @@ -548,10 +573,12 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, { - Name: "Author", + Description: client.CollectionDescription{ + Name: "Author", + Indexes: []client.IndexDescription{}, + }, Schema: client.SchemaDescription{ Name: "Author", Fields: []client.FieldDescription{ @@ -580,7 +607,6 @@ func TestSingleSimpleType(t *testing.T) { }, }, }, - Indexes: []client.IndexDescription{}, }, }, }, @@ -599,12 +625,12 @@ func runCreateDescriptionTest(t *testing.T, testcase descriptionTestCase) { assert.Equal(t, len(descs), len(testcase.targetDescs), testcase.description) for i, d := range descs { - assert.Equal(t, testcase.targetDescs[i], d, testcase.description) + assert.Equal(t, testcase.targetDescs[i].Description, d.Description, testcase.description) } } type descriptionTestCase struct { description string sdl string - targetDescs []client.CollectionDescription + targetDescs []client.CollectionDefinition } diff --git a/request/graphql/schema/generate.go b/request/graphql/schema/generate.go index e30693b3de..5ccc4a897a 100644 --- a/request/graphql/schema/generate.go +++ b/request/graphql/schema/generate.go @@ -47,7 +47,7 @@ func (m *SchemaManager) NewGenerator() *Generator { // Generate generates the query-op and mutation-op type definitions from // the given CollectionDescriptions. -func (g *Generator) Generate(ctx context.Context, collections []client.CollectionDescription) ([]*gql.Object, error) { +func (g *Generator) Generate(ctx context.Context, collections []client.CollectionDefinition) ([]*gql.Object, error) { typeMapBeforeMutation := g.manager.schema.TypeMap() typesBeforeMutation := make(map[string]any, len(typeMapBeforeMutation)) @@ -79,7 +79,7 @@ func (g *Generator) Generate(ctx context.Context, collections []client.Collectio // generate generates the query-op and mutation-op type definitions from // the given CollectionDescriptions. -func (g *Generator) generate(ctx context.Context, collections []client.CollectionDescription) ([]*gql.Object, error) { +func (g *Generator) generate(ctx context.Context, collections []client.CollectionDefinition) ([]*gql.Object, error) { // build base types defs, err := g.buildTypes(ctx, collections) if err != nil { @@ -354,7 +354,7 @@ func (g *Generator) createExpandedFieldList( // extract and return the correct gql.Object type(s) func (g *Generator) buildTypes( ctx context.Context, - collections []client.CollectionDescription, + collections []client.CollectionDefinition, ) ([]*gql.Object, error) { // @todo: Check for duplicate named defined types in the TypeMap // get all the defined types from the AST @@ -367,12 +367,12 @@ func (g *Generator) buildTypes( fieldDescriptions := collection.Schema.Fields // check if type exists - if _, ok := g.manager.schema.TypeMap()[collection.Name]; ok { - return nil, NewErrSchemaTypeAlreadyExist(collection.Name) + if _, ok := g.manager.schema.TypeMap()[collection.Description.Name]; ok { + return nil, NewErrSchemaTypeAlreadyExist(collection.Description.Name) } objconf := gql.ObjectConfig{ - Name: collection.Name, + Name: collection.Description.Name, } // Wrap field definition in a thunk so we can @@ -435,9 +435,9 @@ func (g *Generator) buildTypes( Type: gql.Boolean, } - gqlType, ok := g.manager.schema.TypeMap()[collection.Name] + gqlType, ok := g.manager.schema.TypeMap()[collection.Description.Name] if !ok { - return nil, NewErrObjectNotFoundDuringThunk(collection.Name) + return nil, NewErrObjectNotFoundDuringThunk(collection.Description.Name) } fields[request.GroupFieldName] = &gql.Field{ diff --git a/request/graphql/schema/index_test.go b/request/graphql/schema/index_test.go index 379b84647d..155a17fbf6 100644 --- a/request/graphql/schema/index_test.go +++ b/request/graphql/schema/index_test.go @@ -276,9 +276,9 @@ func parseIndexAndTest(t *testing.T, testCase indexTestCase) { cols, err := FromString(ctx, testCase.sdl) assert.NoError(t, err, testCase.description) assert.Equal(t, len(cols), 1, testCase.description) - assert.Equal(t, len(cols[0].Indexes), len(testCase.targetDescriptions), testCase.description) + assert.Equal(t, len(cols[0].Description.Indexes), len(testCase.targetDescriptions), testCase.description) - for i, d := range cols[0].Indexes { + for i, d := range cols[0].Description.Indexes { assert.Equal(t, testCase.targetDescriptions[i], d, testCase.description) } } diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 261561ca8d..6176273b02 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -200,11 +200,11 @@ func (w *Wrapper) GetCollectionByName(ctx context.Context, name client.Collectio if err != nil { return nil, err } - var colDesc client.CollectionDescription - if err := json.Unmarshal(data, &colDesc); err != nil { + var definition client.CollectionDefinition + if err := json.Unmarshal(data, &definition); err != nil { return nil, err } - return &Collection{w.cmd, colDesc}, nil + return &Collection{w.cmd, definition}, nil } func (w *Wrapper) GetCollectionBySchemaID(ctx context.Context, schemaId string) (client.Collection, error) { @@ -215,11 +215,11 @@ func (w *Wrapper) GetCollectionBySchemaID(ctx context.Context, schemaId string) if err != nil { return nil, err } - var colDesc client.CollectionDescription - if err := json.Unmarshal(data, &colDesc); err != nil { + var definition client.CollectionDefinition + if err := json.Unmarshal(data, &definition); err != nil { return nil, err } - return &Collection{w.cmd, colDesc}, nil + return &Collection{w.cmd, definition}, nil } func (w *Wrapper) GetCollectionByVersionID(ctx context.Context, versionId string) (client.Collection, error) { @@ -230,11 +230,11 @@ func (w *Wrapper) GetCollectionByVersionID(ctx context.Context, versionId string if err != nil { return nil, err } - var colDesc client.CollectionDescription - if err := json.Unmarshal(data, &colDesc); err != nil { + var definition client.CollectionDefinition + if err := json.Unmarshal(data, &definition); err != nil { return nil, err } - return &Collection{w.cmd, colDesc}, nil + return &Collection{w.cmd, definition}, nil } func (w *Wrapper) GetAllCollections(ctx context.Context) ([]client.Collection, error) { @@ -244,7 +244,7 @@ func (w *Wrapper) GetAllCollections(ctx context.Context) ([]client.Collection, e if err != nil { return nil, err } - var colDesc []client.CollectionDescription + var colDesc []client.CollectionDefinition if err := json.Unmarshal(data, &colDesc); err != nil { return nil, err } diff --git a/tests/clients/cli/wrapper_collection.go b/tests/clients/cli/wrapper_collection.go index 3500bdce7c..4f4a97741d 100644 --- a/tests/clients/cli/wrapper_collection.go +++ b/tests/clients/cli/wrapper_collection.go @@ -26,37 +26,41 @@ import ( var _ client.Collection = (*Collection)(nil) type Collection struct { - cmd *cliWrapper - desc client.CollectionDescription + cmd *cliWrapper + def client.CollectionDefinition } func (c *Collection) Description() client.CollectionDescription { - return c.desc + return c.def.Description } func (c *Collection) Name() string { - return c.desc.Name + return c.Description().Name } func (c *Collection) Schema() client.SchemaDescription { - return c.desc.Schema + return c.def.Schema } func (c *Collection) ID() uint32 { - return c.desc.ID + return c.Description().ID } func (c *Collection) SchemaID() string { - return c.desc.Schema.SchemaID + return c.Schema().SchemaID +} + +func (c *Collection) Definition() client.CollectionDefinition { + return c.def } func (c *Collection) Create(ctx context.Context, doc *client.Document) error { args := []string{"client", "collection", "create"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) // We must call this here, else the doc key on the given object will not match // that of the document saved in the database - err := doc.RemapAliasFieldsAndDockey(c.Description().Schema.Fields) + err := doc.RemapAliasFieldsAndDockey(c.Schema().Fields) if err != nil { return err } @@ -76,13 +80,13 @@ func (c *Collection) Create(ctx context.Context, doc *client.Document) error { func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) error { args := []string{"client", "collection", "create"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) docMapList := make([]map[string]any, len(docs)) for i, doc := range docs { // We must call this here, else the doc key on the given object will not match // that of the document saved in the database - err := doc.RemapAliasFieldsAndDockey(c.Description().Schema.Fields) + err := doc.RemapAliasFieldsAndDockey(c.Schema().Fields) if err != nil { return err } @@ -110,7 +114,7 @@ func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) er func (c *Collection) Update(ctx context.Context, doc *client.Document) error { args := []string{"client", "collection", "update"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) args = append(args, "--key", doc.Key().String()) document, err := doc.ToJSONPatch() @@ -188,7 +192,7 @@ func (c *Collection) UpdateWithFilter( updater string, ) (*client.UpdateResult, error) { args := []string{"client", "collection", "update"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) args = append(args, "--updater", updater) filterJSON, err := json.Marshal(filter) @@ -206,7 +210,7 @@ func (c *Collection) UpdateWithKey( updater string, ) (*client.UpdateResult, error) { args := []string{"client", "collection", "update"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) args = append(args, "--key", key.String()) args = append(args, "--updater", updater) @@ -219,7 +223,7 @@ func (c *Collection) UpdateWithKeys( updater string, ) (*client.UpdateResult, error) { args := []string{"client", "collection", "update"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) args = append(args, "--updater", updater) keys := make([]string, len(docKeys)) @@ -261,7 +265,7 @@ func (c *Collection) deleteWith( func (c *Collection) DeleteWithFilter(ctx context.Context, filter any) (*client.DeleteResult, error) { args := []string{"client", "collection", "delete"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) filterJSON, err := json.Marshal(filter) if err != nil { @@ -274,7 +278,7 @@ func (c *Collection) DeleteWithFilter(ctx context.Context, filter any) (*client. func (c *Collection) DeleteWithKey(ctx context.Context, docKey client.DocKey) (*client.DeleteResult, error) { args := []string{"client", "collection", "delete"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) args = append(args, "--key", docKey.String()) return c.deleteWith(ctx, args) @@ -282,7 +286,7 @@ func (c *Collection) DeleteWithKey(ctx context.Context, docKey client.DocKey) (* func (c *Collection) DeleteWithKeys(ctx context.Context, docKeys []client.DocKey) (*client.DeleteResult, error) { args := []string{"client", "collection", "delete"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) keys := make([]string, len(docKeys)) for i, v := range docKeys { @@ -295,7 +299,7 @@ func (c *Collection) DeleteWithKeys(ctx context.Context, docKeys []client.DocKey func (c *Collection) Get(ctx context.Context, key client.DocKey, showDeleted bool) (*client.Document, error) { args := []string{"client", "collection", "get"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) args = append(args, key.String()) if showDeleted { @@ -315,14 +319,14 @@ func (c *Collection) Get(ctx context.Context, key client.DocKey, showDeleted boo func (c *Collection) WithTxn(tx datastore.Txn) client.Collection { return &Collection{ - cmd: c.cmd.withTxn(tx), - desc: c.desc, + cmd: c.cmd.withTxn(tx), + def: c.def, } } func (c *Collection) GetAllDocKeys(ctx context.Context) (<-chan client.DocKeysResult, error) { args := []string{"client", "collection", "keys"} - args = append(args, "--name", c.desc.Name) + args = append(args, "--name", c.Description().Name) stdOut, _, err := c.cmd.executeStream(ctx, args) if err != nil { @@ -361,7 +365,7 @@ func (c *Collection) CreateIndex( indexDesc client.IndexDescription, ) (index client.IndexDescription, err error) { args := []string{"client", "index", "create"} - args = append(args, "--collection", c.desc.Name) + args = append(args, "--collection", c.Description().Name) args = append(args, "--name", indexDesc.Name) fields := make([]string, len(indexDesc.Fields)) @@ -382,7 +386,7 @@ func (c *Collection) CreateIndex( func (c *Collection) DropIndex(ctx context.Context, indexName string) error { args := []string{"client", "index", "drop"} - args = append(args, "--collection", c.desc.Name) + args = append(args, "--collection", c.Description().Name) args = append(args, "--name", indexName) _, err := c.cmd.execute(ctx, args) @@ -391,7 +395,7 @@ func (c *Collection) DropIndex(ctx context.Context, indexName string) error { func (c *Collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { args := []string{"client", "index", "list"} - args = append(args, "--collection", c.desc.Name) + args = append(args, "--collection", c.Description().Name) data, err := c.cmd.execute(ctx, args) if err != nil { diff --git a/tests/integration/schema/updates/remove/simple_test.go b/tests/integration/schema/updates/remove/simple_test.go index 1bbb956f4b..19f9ea1836 100644 --- a/tests/integration/schema/updates/remove/simple_test.go +++ b/tests/integration/schema/updates/remove/simple_test.go @@ -143,7 +143,7 @@ func TestSchemaUpdatesRemoveSchemaNameErrors(t *testing.T) { { "op": "remove", "path": "/Users/Schema/Name" } ] `, - ExpectedError: "modifying the schema name is not supported. ExistingName: Users, ProposedName: ", + ExpectedError: "schema name can't be empty", }, }, }