feat: Add materialized views

.github/workflows/test-and-upload-coverage.yml

@@ -37,6 +37,7 @@ jobs:
         lens-type: [wasm-time]
         acp-type: [local]
         database-encryption: [false]
+        view-type: [cacheless]
         include:
           - os: ubuntu-latest
             client-type: go
@@ -45,48 +46,63 @@ jobs:
             lens-type: wasm-time
             acp-type: local
             database-encryption: true
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wazero
             acp-type: local
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasmer
             acp-type: local
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: source-hub
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: http
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: source-hub
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: cli
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: source-hub
             database-encryption: false
+            view-type: cacheless
+          - os: ubuntu-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            lens-type: wasm-time
+            acp-type: local
+            database-encryption: false
+            view-type: materialized
           - os: macos-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: local
             database-encryption: false
+            view-type: cacheless
 ## TODO: https://github.com/sourcenetwork/defradb/issues/2080
 ## Uncomment the lines below to Re-enable the windows build once this todo is resolved.
 ##        - os: windows-latest
@@ -96,6 +112,7 @@ jobs:
 ##          lens-type: wasm-time
 ##          acp-type: local
 ##          database-encryption: false
+##          view-type: cacheless
 
     runs-on: ${{ matrix.os }}
 
@@ -115,6 +132,7 @@ jobs:
       DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }}
       DEFRA_LENS_TYPE: ${{ matrix.lens-type }}
       DEFRA_ACP_TYPE: ${{ matrix.acp-type }}
+      DEFRA_VIEW_TYPE: ${{ matrix.view-type }}
 
     steps:
       - name: Checkout code into the directory

cli/cli.go

@@ -75,6 +75,7 @@ func NewDefraCommand() *cobra.Command {
 	view := MakeViewCommand()
 	view.AddCommand(
 		MakeViewAddCommand(),
+		MakeViewRefreshCommand(),
 	)
 
 	index := MakeIndexCommand()

cli/view_refresh.go

@@ -0,0 +1,75 @@
+// Copyright 2024 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package cli
+
+import (
+	"github.com/sourcenetwork/immutable"
+	"github.com/spf13/cobra"
+
+	"github.com/sourcenetwork/defradb/client"
+)
+
+func MakeViewRefreshCommand() *cobra.Command {
+	var name string
+	var schemaRoot string
+	var versionID string
+	var getInactive bool
+	var cmd = &cobra.Command{
+		Use:   "refresh",
+		Short: "Refresh views.",
+		Long: `Refresh views, executing the underlying query and LensVm transforms and
+persisting the results.
+
+View is refreshed as the current user, meaning the cached items will reflect that user's
+permissions. Subsequent query requests to the view, regardless of user, will receive
+items from that cache.
+
+Example: refresh all views
+  defradb client view refresh
+
+Example: refresh views by name
+  defradb client view refresh --name UserView
+
+Example: refresh views by schema root id
+  defradb client view refresh --schema bae123
+
+Example: refresh views by version id. This will also return inactive views
+  defradb client view refresh --version bae123
+		`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			store := mustGetContextStore(cmd)
+
+			options := client.CollectionFetchOptions{}
+			if versionID != "" {
+				options.SchemaVersionID = immutable.Some(versionID)
+			}
+			if schemaRoot != "" {
+				options.SchemaRoot = immutable.Some(schemaRoot)
+			}
+			if name != "" {
+				options.Name = immutable.Some(name)
+			}
+			if getInactive {
+				options.IncludeInactive = immutable.Some(getInactive)
+			}
+
+			return store.RefreshViews(
+				cmd.Context(),
+				options,
+			)
+		},
+	}
+	cmd.Flags().StringVar(&name, "name", "", "View name")
+	cmd.Flags().StringVar(&schemaRoot, "schema", "", "View schema Root")
+	cmd.Flags().StringVar(&versionID, "version", "", "View version ID")
+	cmd.Flags().BoolVar(&getInactive, "get-inactive", false, "Get inactive views as well as active")
+	return cmd
+}

client/collection_description.go

@@ -79,6 +79,15 @@ type CollectionDescription struct {
 	// parsing is done, to avoid storing an invalid policyID or policy resource
 	// that may not even exist on acp.
 	Policy immutable.Option[PolicyDescription]
+
+	// IsMaterialized defines whether the items in this collection are cached or not.
+	//
+	// If it is true, they will be, if false, the data returned on query will be calculated
+	// at query-time from source.
+	//
+	// At the moment this can only be set to `false` if this collection sources its data from
+	// another collection/query (is a View).
+	IsMaterialized bool
 }
 
 // QuerySource represents a collection data source from a query.
@@ -179,6 +188,7 @@ type collectionDescription struct {
 	ID              uint32
 	RootID          uint32
 	SchemaVersionID string
+	IsMaterialized  bool
 	Policy          immutable.Option[PolicyDescription]
 	Indexes         []IndexDescription
 	Fields          []CollectionFieldDescription
@@ -198,6 +208,7 @@ func (c *CollectionDescription) UnmarshalJSON(bytes []byte) error {
 	c.ID = descMap.ID
 	c.RootID = descMap.RootID
 	c.SchemaVersionID = descMap.SchemaVersionID
+	c.IsMaterialized = descMap.IsMaterialized
 	c.Indexes = descMap.Indexes
 	c.Fields = descMap.Fields
 	c.Sources = make([]any, len(descMap.Sources))

client/db.go

@@ -196,6 +196,14 @@ type Store interface {
 		transform immutable.Option[model.Lens],
 	) ([]CollectionDefinition, error)
 
+	// RefreshViews refreshes the caches of all views matching the given options.  If no options are set, all views
+	// will be refreshed.
+	//
+	// The cached result is dependent on the ACP settings of the source data and the permissions of the user making
+	// the call.  At the moment only one cache can be active at a time, so please pay attention to access rights
+	// when making this call.
+	RefreshViews(context.Context, CollectionFetchOptions) error
+
 	// SetMigration sets the migration for all collections using the given source-destination schema version IDs.
 	//
 	// There may only be one migration per collection version.  If another migration was registered it will be

docs/data_format_changes/i2951-no-change-tests-updated.md

@@ -0,0 +1,3 @@
+# Materialized views
+
+Views have been made materialized by default, this caused the tests to change slightly.

docs/website/references/cli/defradb_client_view.md

@@ -37,4 +37,5 @@ Manage (add) views withing a running DefraDB instance
 
 * [defradb client](defradb_client.md)	 - Interact with a DefraDB node
 * [defradb client view add](defradb_client_view_add.md)	 - Add new view
+* [defradb client view refresh](defradb_client_view_refresh.md)	 - Refresh views.
 

docs/website/references/cli/defradb_client_view_refresh.md

@@ -0,0 +1,65 @@
+## defradb client view refresh
+
+Refresh views.
+
+### Synopsis
+
+Refresh views, executing the underlying query and LensVm transforms and
+persisting the results.
+
+View is refreshed as the current user, meaning the cached items will reflect that user's
+permissions. Subsequent query requests to the view, regardless of user, will receive
+items from that cache.
+
+Example: refresh all views
+  defradb client view refresh
+
+Example: refresh views by name
+  defradb client view refresh --name UserView
+
+Example: refresh views by schema root id
+  defradb client view refresh --schema bae123
+
+Example: refresh views by version id. This will also return inactive views
+  defradb client view refresh --version bae123
+
+
+```
+defradb client view refresh [flags]
+```
+
+### Options
+
+```
+      --get-inactive     Get inactive views as well as active
+  -h, --help             help for refresh
+      --name string      View name
+      --schema string    View schema Root
+      --version string   View version ID
+```
+
+### Options inherited from parent commands
+
+```
+  -i, --identity string             Hex formatted private key used to authenticate with ACP
+      --keyring-backend string      Keyring backend to use. Options are file or system (default "file")
+      --keyring-namespace string    Service name to use when using the system backend (default "defradb")
+      --keyring-path string         Path to store encrypted keys when using the file backend (default "keys")
+      --log-format string           Log format to use. Options are text or json (default "text")
+      --log-level string            Log level to use. Options are debug, info, error, fatal (default "info")
+      --log-output string           Log output path. Options are stderr or stdout. (default "stderr")
+      --log-overrides string        Logger config overrides. Format <name>,<key>=<val>,...;<name>,...
+      --log-source                  Include source location in logs
+      --log-stacktrace              Include stacktrace in error and fatal logs
+      --no-keyring                  Disable the keyring and generate ephemeral keys
+      --no-log-color                Disable colored log output
+      --rootdir string              Directory for persistent data (default: $HOME/.defradb)
+      --source-hub-address string   The SourceHub address authorized by the client to make SourceHub transactions on behalf of the actor
+      --tx uint                     Transaction ID
+      --url string                  URL of HTTP endpoint to listen on or connect to (default "127.0.0.1:9181")
+```
+
+### SEE ALSO
+
+* [defradb client view](defradb_client_view.md)	 - Manage views within a running DefraDB instance
+