Add cached views

.github/workflows/test-and-upload-coverage.yml

@@ -37,6 +37,7 @@ jobs:
         lens-type: [wasm-time]
         acp-type: [local]
         database-encryption: [false]
+        view-type: [cacheless]
         include:
           - os: ubuntu-latest
             client-type: go
@@ -45,48 +46,63 @@ jobs:
             lens-type: wasm-time
             acp-type: local
             database-encryption: true
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wazero
             acp-type: local
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasmer
             acp-type: local
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: source-hub
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: http
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: source-hub
             database-encryption: false
+            view-type: cacheless
           - os: ubuntu-latest
             client-type: cli
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: source-hub
             database-encryption: false
+            view-type: cacheless
+          - os: ubuntu-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            lens-type: wasm-time
+            acp-type: local
+            database-encryption: false
+            view-type: materialized
           - os: macos-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
             lens-type: wasm-time
             acp-type: local
             database-encryption: false
+            view-type: cacheless
 ## TODO: https://github.com/sourcenetwork/defradb/issues/2080
 ## Uncomment the lines below to Re-enable the windows build once this todo is resolved.
 ##        - os: windows-latest
@@ -96,6 +112,7 @@ jobs:
 ##          lens-type: wasm-time
 ##          acp-type: local
 ##          database-encryption: false
+##          view-type: cacheless
 
     runs-on: ${{ matrix.os }}
 
@@ -115,6 +132,7 @@ jobs:
       DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }}
       DEFRA_LENS_TYPE: ${{ matrix.lens-type }}
       DEFRA_ACP_TYPE: ${{ matrix.acp-type }}
+      DEFRA_VIEW_TYPE: ${{ matrix.view-type }}
 
     steps:
       - name: Checkout code into the directory

cli/cli.go

@@ -75,6 +75,7 @@ func NewDefraCommand() *cobra.Command {
 	view := MakeViewCommand()
 	view.AddCommand(
 		MakeViewAddCommand(),
+		MakeViewRefreshCommand(),
 	)
 
 	index := MakeIndexCommand()

cli/view_refresh.go

@@ -0,0 +1,74 @@
+// Copyright 2024 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package cli
+
+import (
+	"github.com/sourcenetwork/immutable"
+	"github.com/spf13/cobra"
+
+	"github.com/sourcenetwork/defradb/client"
+)
+
+func MakeViewRefreshCommand() *cobra.Command {
+	var name string
+	var schemaRoot string
+	var versionID string
+	var getInactive bool
+	var cmd = &cobra.Command{
+		Use:   "refresh",
+		Short: "Refresh views.",
+		Long: `Refresh views, executing the underlying query and LensVm transforms and
+persisting the results.
+
+View is refreshed as the current user, meaning results returned for all subsequent query requests
+to the view will recieve items accessible to the user refreshing the view's permissions.
+
+Example: refresh all views
+  defradb view refresh
+
+Example: refresh views by name
+  defradb view refresh --name UserView
+
+Example: refresh views by schema root id
+  defradb view refresh --schema bae123
+
+Example: refresh views by version id. This will also return inactive views
+  defradb view refresh --version bae123
+		`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			store := mustGetContextStore(cmd)
+
+			options := client.CollectionFetchOptions{}
+			if versionID != "" {
+				options.SchemaVersionID = immutable.Some(versionID)
+			}
+			if schemaRoot != "" {
+				options.SchemaRoot = immutable.Some(schemaRoot)
+			}
+			if name != "" {
+				options.Name = immutable.Some(name)
+			}
+			if getInactive {
+				options.IncludeInactive = immutable.Some(getInactive)
+			}
+
+			return store.RefreshViews(
+				cmd.Context(),
+				options,
+			)
+		},
+	}
+	cmd.Flags().StringVar(&name, "name", "", "View name")
+	cmd.Flags().StringVar(&schemaRoot, "schema", "", "View schema Root")
+	cmd.Flags().StringVar(&versionID, "version", "", "View version ID")
+	cmd.Flags().BoolVar(&getInactive, "get-inactive", false, "Get inactive views as well as active")
+	return cmd
+}

client/collection_description.go

@@ -79,6 +79,15 @@ type CollectionDescription struct {
 	// parsing is done, to avoid storing an invalid policyID or policy resource
 	// that may not even exist on acp.
 	Policy immutable.Option[PolicyDescription]
+
+	// IsMaterialized defines whether the items in this collection are cached or not.
+	//
+	// If it is true, they will be, if false, the data returned on query will be calculated
+	// at query-time from source.
+	//
+	// At the moment this can only be set to `false` if this collection sources it's data from
+	// another collection/query (is a View).
+	IsMaterialized bool
 }
 
 // QuerySource represents a collection data source from a query.
@@ -179,6 +188,7 @@ type collectionDescription struct {
 	ID              uint32
 	RootID          uint32
 	SchemaVersionID string
+	IsMaterialized  bool
 	Policy          immutable.Option[PolicyDescription]
 	Indexes         []IndexDescription
 	Fields          []CollectionFieldDescription
@@ -198,6 +208,7 @@ func (c *CollectionDescription) UnmarshalJSON(bytes []byte) error {
 	c.ID = descMap.ID
 	c.RootID = descMap.RootID
 	c.SchemaVersionID = descMap.SchemaVersionID
+	c.IsMaterialized = descMap.IsMaterialized
 	c.Indexes = descMap.Indexes
 	c.Fields = descMap.Fields
 	c.Sources = make([]any, len(descMap.Sources))

client/db.go

@@ -196,6 +196,14 @@ type Store interface {
 		transform immutable.Option[model.Lens],
 	) ([]CollectionDefinition, error)
 
+	// RefreshViews refreshes the caches of all views matching the given options.  If no options are set all views
+	// will be refreshed.
+	//
+	// The cached result is dependent on the ACP settings of the source data and the permissions of the user making
+	// the call.  At the moment only one cache can be active at a time, so please pay attention to access rights
+	// when making this call.
+	RefreshViews(context.Context, CollectionFetchOptions) error
+
 	// SetMigration sets the migration for all collections using the given source-destination schema version IDs.
 	//
 	// There may only be one migration per collection version.  If another migration was registered it will be

http/client.go

@@ -219,6 +219,32 @@ func (c *Client) AddView(
 	return descriptions, nil
 }
 
+func (c *Client) RefreshViews(ctx context.Context, options client.CollectionFetchOptions) error {
+	methodURL := c.http.baseURL.JoinPath("view", "refresh")
+	params := url.Values{}
+	if options.Name.HasValue() {
+		params.Add("name", options.Name.Value())
+	}
+	if options.SchemaVersionID.HasValue() {
+		params.Add("version_id", options.SchemaVersionID.Value())
+	}
+	if options.SchemaRoot.HasValue() {
+		params.Add("schema_root", options.SchemaRoot.Value())
+	}
+	if options.IncludeInactive.HasValue() {
+		params.Add("get_inactive", strconv.FormatBool(options.IncludeInactive.Value()))
+	}
+	methodURL.RawQuery = params.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, methodURL.String(), nil)
+	if err != nil {
+		return err
+	}
+
+	_, err = c.http.request(req)
+	return err
+}
+
 func (c *Client) SetMigration(ctx context.Context, config client.LensConfig) error {
 	methodURL := c.http.baseURL.JoinPath("lens")
 

http/handler_store.go

@@ -220,6 +220,38 @@ func (s *storeHandler) GetSchema(rw http.ResponseWriter, req *http.Request) {
 	responseJSON(rw, http.StatusOK, schema)
 }
 
+func (s *storeHandler) RefreshViews(rw http.ResponseWriter, req *http.Request) {
+	store := req.Context().Value(dbContextKey).(client.Store)
+
+	options := client.CollectionFetchOptions{}
+	if req.URL.Query().Has("name") {
+		options.Name = immutable.Some(req.URL.Query().Get("name"))
+	}
+	if req.URL.Query().Has("version_id") {
+		options.SchemaVersionID = immutable.Some(req.URL.Query().Get("version_id"))
+	}
+	if req.URL.Query().Has("schema_root") {
+		options.SchemaRoot = immutable.Some(req.URL.Query().Get("schema_root"))
+	}
+	if req.URL.Query().Has("get_inactive") {
+		getInactiveStr := req.URL.Query().Get("get_inactive")
+		var err error
+		getInactive, err := strconv.ParseBool(getInactiveStr)
+		if err != nil {
+			responseJSON(rw, http.StatusBadRequest, errorResponse{err})
+			return
+		}
+		options.IncludeInactive = immutable.Some(getInactive)
+	}
+
+	err := store.RefreshViews(req.Context(), options)
+	if err != nil {
+		responseJSON(rw, http.StatusBadRequest, errorResponse{err})
+		return
+	}
+	rw.WriteHeader(http.StatusOK)
+}
+
 func (s *storeHandler) GetAllIndexes(rw http.ResponseWriter, req *http.Request) {
 	store := req.Context().Value(dbContextKey).(client.Store)
 
@@ -482,6 +514,18 @@ func (h *storeHandler) bindRoutes(router *Router) {
 	collectionDescribe.AddResponse(200, collectionsResponse)
 	collectionDescribe.Responses.Set("400", errorResponse)
 
+	viewRefresh := openapi3.NewOperation()
+	viewRefresh.OperationID = "view_refresh"
+	viewRefresh.Description = "Refresh view(s) by name, schema id, or version id."
+	viewRefresh.Tags = []string{"view"}
+	viewRefresh.AddParameter(collectionNameQueryParam)
+	viewRefresh.AddParameter(collectionSchemaRootQueryParam)
+	viewRefresh.AddParameter(collectionVersionIdQueryParam)
+	viewRefresh.AddParameter(collectionGetInactiveQueryParam)
+	viewRefresh.Responses = openapi3.NewResponses()
+	viewRefresh.Responses.Set("200", successResponse)
+	viewRefresh.Responses.Set("400", errorResponse)
+
 	patchCollection := openapi3.NewOperation()
 	patchCollection.OperationID = "patch_collection"
 	patchCollection.Description = "Update collection definitions"
@@ -609,6 +653,7 @@ func (h *storeHandler) bindRoutes(router *Router) {
 	router.AddRoute("/collections", http.MethodGet, collectionDescribe, h.GetCollection)
 	router.AddRoute("/collections", http.MethodPatch, patchCollection, h.PatchCollection)
 	router.AddRoute("/view", http.MethodPost, views, h.AddView)
+	router.AddRoute("/view/refresh", http.MethodPost, viewRefresh, h.RefreshViews)
 	router.AddRoute("/graphql", http.MethodGet, graphQLGet, h.ExecRequest)
 	router.AddRoute("/graphql", http.MethodPost, graphQLPost, h.ExecRequest)
 	router.AddRoute("/debug/dump", http.MethodGet, debugDump, h.PrintDump)