[full-ci] Rescan spaces

.drone.env

@@ -1,5 +1,5 @@
 # The test runner source for API tests
-CORE_COMMITID=a4a490bbebdc625c328c816e5f3d09fcd94550a9
+CORE_COMMITID=6a3c89e917330990f99feabef3b42d1c51c0df1d
 CORE_BRANCH=master
 
 # The test runner source for UI tests

changelog/unreleased/rescan-spaces.md

@@ -0,0 +1,7 @@
+Bugfix: Trigger a rescan of spaces in the search index when items have changed
+
+The search service now scans spaces when items have been changed. This fixes the problem
+that mtime and treesize propagation was not reflected in the search index properly.
+
+https://github.com/owncloud/ocis/pull/4777
+https://github.com/owncloud/ocis/issues/4410

go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/armon/go-radix v1.0.0
 	github.com/blevesearch/bleve/v2 v2.3.4
 	github.com/coreos/go-oidc/v3 v3.4.0
-	github.com/cs3org/go-cs3apis v0.0.0-20221005085457-19ea8088a512
-	github.com/cs3org/reva/v2 v2.10.1-0.20221012104058-ae7c58b9bffa
+	github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965
+	github.com/cs3org/reva/v2 v2.10.1-0.20221013183308-560ba925a814
 	github.com/disintegration/imaging v1.6.2
 	github.com/ggwhite/go-masker v1.0.9
 	github.com/go-chi/chi/v5 v5.0.7

go.sum

@@ -349,10 +349,10 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo
 github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4=
 github.com/crewjam/saml v0.4.6 h1:XCUFPkQSJLvzyl4cW9OvpWUbRf0gE7VUpU8ZnilbeM4=
 github.com/crewjam/saml v0.4.6/go.mod h1:ZBOXnNPFzB3CgOkRm7Nd6IVdkG+l/wF+0ZXLqD96t1A=
-github.com/cs3org/go-cs3apis v0.0.0-20221005085457-19ea8088a512 h1:xTvaIsLu1ezoWOJKnV0ehgiowkOiEhMaylaI1lD/Axw=
-github.com/cs3org/go-cs3apis v0.0.0-20221005085457-19ea8088a512/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
-github.com/cs3org/reva/v2 v2.10.1-0.20221012104058-ae7c58b9bffa h1:DSeaakiPW5zYrGGEDO0BkSZWhqq6LS+rd1DQ1DPztJo=
-github.com/cs3org/reva/v2 v2.10.1-0.20221012104058-ae7c58b9bffa/go.mod h1:QUHLTf/ACFG2ueNP3u1dslv1bIWTTQAqvWFCorVke6o=
+github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965 h1:y4n2j68LLnvac+zw/al8MfPgO5aQiIwLmHM/JzYN8AM=
+github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
+github.com/cs3org/reva/v2 v2.10.1-0.20221013183308-560ba925a814 h1:/IpXuGNX01f2eKM5mSMJFoE+DGq4NULo0WwpO3LzmTg=
+github.com/cs3org/reva/v2 v2.10.1-0.20221013183308-560ba925a814/go.mod h1:lq+LRpBDYU1vHUmJDeK7sGquREciO8GDj5/SYIibMPY=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

services/search/pkg/command/index.go

@@ -3,9 +3,12 @@ package command
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/urfave/cli/v2"
+	"go-micro.dev/v4/client"
 
+	"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
 	"github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
 	searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0"
 	"github.com/owncloud/ocis/v2/services/search/pkg/config"
@@ -30,18 +33,20 @@ func Index(cfg *config.Config) *cli.Command {
 				Name:     "user",
 				Aliases:  []string{"u"},
 				Required: true,
-				Usage:    "the username of the user tha shall be used to access the files",
+				Usage:    "the username of the user that shall be used to access the files",
 			},
 		},
 		Before: func(c *cli.Context) error {
-			return parser.ParseConfig(cfg)
+			return configlog.ReturnFatal(parser.ParseConfig(cfg))
 		},
-		Action: func(c *cli.Context) error {
-			client := searchsvc.NewSearchProviderService("com.owncloud.api.search", grpc.DefaultClient())
-			_, err := client.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{
-				SpaceId: c.String("space"),
-				UserId:  c.String("user"),
-			})
+		Action: func(ctx *cli.Context) error {
+			grpcClient := grpc.DefaultClient()
+			grpcClient.Options()
+			c := searchsvc.NewSearchProviderService("com.owncloud.api.search", grpcClient)
+			_, err := c.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{
+				SpaceId: ctx.String("space"),
+				UserId:  ctx.String("user"),
+			}, func(opts *client.CallOptions) { opts.RequestTimeout = 10 * time.Minute })
 			if err != nil {
 				fmt.Println("failed to index space: " + err.Error())
 				return err

services/search/pkg/search/index/index.go

@@ -222,13 +222,18 @@ func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (
 	query := bleve.NewConjunctionQuery(
 		bleve.NewQueryStringQuery(req.Query),
 		deletedQuery, // Skip documents that have been marked as deleted
-		bleve.NewQueryStringQuery("RootID:"+idToBleveId(&sprovider.ResourceId{
-			StorageId: req.Ref.GetResourceId().GetStorageId(),
-			SpaceId:   req.Ref.GetResourceId().GetSpaceId(),
-			OpaqueId:  req.Ref.GetResourceId().GetOpaqueId(),
-		})), // Limit search to the space
-		bleve.NewQueryStringQuery("Path:"+queryEscape(utils.MakeRelativePath(path.Join(req.Ref.Path, "/"))+"*")), // Limit search to this directory in the space
 	)
+	if req.Ref != nil {
+		query = bleve.NewConjunctionQuery(
+			query,
+			bleve.NewQueryStringQuery("RootID:"+idToBleveId(&sprovider.ResourceId{
+				StorageId: req.Ref.GetResourceId().GetStorageId(),
+				SpaceId:   req.Ref.GetResourceId().GetSpaceId(),
+				OpaqueId:  req.Ref.GetResourceId().GetOpaqueId(),
+			})), // Limit search to the space
+			bleve.NewQueryStringQuery("Path:"+queryEscape(utils.MakeRelativePath(path.Join(req.Ref.Path, "/"))+"*")), // Limit search to this directory in the space
+		)
+	}
 	bleveReq := bleve.NewSearchRequest(query)
 	bleveReq.Size = 200
 	if req.PageSize > 0 {
@@ -295,7 +300,7 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocume
 	}
 
 	if ri.Mtime != nil {
-		doc.Mtime = time.Unix(int64(ri.Mtime.Seconds), int64(ri.Mtime.Nanos)).UTC().Format(time.RFC3339)
+		doc.Mtime = time.Unix(int64(ri.Mtime.Seconds), int64(ri.Mtime.Nanos)).UTC().Format(time.RFC3339Nano)
 	}
 
 	return doc
@@ -350,7 +355,7 @@ func fromDocumentMatch(hit *search.DocumentMatch) (*searchmsg.Match, error) {
 		match.Entity.ParentId = resourceIDtoSearchID(parentID)
 	}
 
-	if mtime, err := time.Parse(time.RFC3339, hit.Fields["Mtime"].(string)); err == nil {
+	if mtime, err := time.Parse(time.RFC3339Nano, hit.Fields["Mtime"].(string)); err == nil {
 		match.Entity.LastModifiedTime = &timestamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
 	}
 

services/search/pkg/search/provider/events.go

@@ -2,6 +2,8 @@ package provider
 
 import (
 	"context"
+	"sync"
+	"time"
 
 	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
 	user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
@@ -11,38 +13,68 @@ import (
 	ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
 	"github.com/cs3org/reva/v2/pkg/errtypes"
 	"github.com/cs3org/reva/v2/pkg/events"
+	"github.com/cs3org/reva/v2/pkg/storagespace"
 	"google.golang.org/grpc/metadata"
 )
 
+// SpaceDebouncer debounces operations on spaces for a configurable amount of time
+type SpaceDebouncer struct {
+	after   time.Duration
+	f       func(id *provider.StorageSpaceId, userID *user.UserId)
+	pending map[string]*time.Timer
+
+	mutex sync.Mutex
+}
+
+// NewSpaceDebouncer returns a new SpaceDebouncer instance
+func NewSpaceDebouncer(d time.Duration, f func(id *provider.StorageSpaceId, userID *user.UserId)) *SpaceDebouncer {
+	return &SpaceDebouncer{
+		after:   d,
+		f:       f,
+		pending: map[string]*time.Timer{},
+	}
+}
+
+// Debounce restars the debounce timer for the given space
+func (d *SpaceDebouncer) Debounce(id *provider.StorageSpaceId, userID *user.UserId) {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	if t := d.pending[id.OpaqueId]; t != nil {
+		t.Stop()
+	}
+
+	d.pending[id.OpaqueId] = time.AfterFunc(d.after, func() {
+		d.f(id, userID)
+	})
+}
+
 func (p *Provider) handleEvent(ev interface{}) {
-	var ref *provider.Reference
-	var owner *user.User
 	switch e := ev.(type) {
 	case events.ItemTrashed:
 		p.logger.Debug().Interface("event", ev).Msg("marking document as deleted")
 		err := p.indexClient.Delete(e.ID)
 		if err != nil {
 			p.logger.Error().Err(err).Interface("Id", e.ID).Msg("failed to remove item from index")
 		}
-		return
+		p.reindexSpace(ev, e.Ref, e.Executant, e.SpaceOwner)
 	case events.ItemRestored:
 		p.logger.Debug().Interface("event", ev).Msg("marking document as restored")
-		ref = e.Ref
-		owner = &user.User{
+		owner := &user.User{
 			Id: e.Executant,
 		}
 
 		ownerCtx, err := p.getAuthContext(owner)
 		if err != nil {
 			return
 		}
-		statRes, err := p.statResource(ownerCtx, ref, owner)
+		statRes, err := p.statResource(ownerCtx, e.Ref, owner)
 		if err != nil {
 			p.logger.Error().Err(err).
-				Str("storageid", ref.GetResourceId().GetStorageId()).
-				Str("spaceid", ref.GetResourceId().GetSpaceId()).
-				Str("opaqueid", ref.GetResourceId().GetOpaqueId()).
-				Str("path", ref.GetPath()).
+				Str("storageid", e.Ref.GetResourceId().GetStorageId()).
+				Str("spaceid", e.Ref.GetResourceId().GetSpaceId()).
+				Str("opaqueid", e.Ref.GetResourceId().GetOpaqueId()).
+				Str("path", e.Ref.GetPath()).
 				Msg("failed to make stat call for the restored resource")
 			return
 		}
@@ -52,34 +84,32 @@ func (p *Provider) handleEvent(ev interface{}) {
 			err = p.indexClient.Restore(statRes.Info.Id)
 			if err != nil {
 				p.logger.Error().Err(err).
-					Str("storageid", ref.GetResourceId().GetStorageId()).
-					Str("spaceid", ref.GetResourceId().GetSpaceId()).
-					Str("opaqueid", ref.GetResourceId().GetOpaqueId()).
-					Str("path", ref.GetPath()).
+					Str("storageid", e.Ref.GetResourceId().GetStorageId()).
+					Str("spaceid", e.Ref.GetResourceId().GetSpaceId()).
+					Str("opaqueid", e.Ref.GetResourceId().GetOpaqueId()).
+					Str("path", e.Ref.GetPath()).
 					Msg("failed to restore the changed resource in the index")
 			}
 		default:
 			p.logger.Error().Interface("statRes", statRes).
-				Str("storageid", ref.GetResourceId().GetStorageId()).
-				Str("spaceid", ref.GetResourceId().GetSpaceId()).
-				Str("opaqueid", ref.GetResourceId().GetOpaqueId()).
-				Str("path", ref.GetPath()).
+				Str("storageid", e.Ref.GetResourceId().GetStorageId()).
+				Str("spaceid", e.Ref.GetResourceId().GetSpaceId()).
+				Str("opaqueid", e.Ref.GetResourceId().GetOpaqueId()).
+				Str("path", e.Ref.GetPath()).
 				Msg("failed to stat the restored resource")
 		}
-
-		return
+		p.reindexSpace(ev, e.Ref, e.Executant, e.SpaceOwner)
 	case events.ItemMoved:
 		p.logger.Debug().Interface("event", ev).Msg("resource has been moved, updating the document")
-		ref = e.Ref
-		owner = &user.User{
+		owner := &user.User{
 			Id: e.Executant,
 		}
 
 		ownerCtx, err := p.getAuthContext(owner)
 		if err != nil {
 			return
 		}
-		statRes, err := p.statResource(ownerCtx, ref, owner)
+		statRes, err := p.statResource(ownerCtx, e.Ref, owner)
 		if err != nil {
 			p.logger.Error().Err(err).Msg("failed to stat the moved resource")
 			return
@@ -91,80 +121,46 @@ func (p *Provider) handleEvent(ev interface{}) {
 
 		gpRes, err := p.getPath(ownerCtx, statRes.Info.Id, owner)
 		if err != nil {
-			p.logger.Error().Err(err).Interface("ref", ref).Msg("failed to get path for moved resource")
+			p.logger.Error().Err(err).Interface("ref", e.Ref).Msg("failed to get path for moved resource")
 			return
 		}
 		if gpRes.Status.Code != rpcv1beta1.Code_CODE_OK {
-			p.logger.Error().Interface("status", gpRes.Status).Interface("ref", ref).Msg("failed to get path for moved resource")
+			p.logger.Error().Interface("status", gpRes.Status).Interface("ref", e.Ref).Msg("failed to get path for moved resource")
 			return
 		}
 
 		err = p.indexClient.Move(statRes.GetInfo().GetId(), statRes.GetInfo().GetParentId(), gpRes.Path)
 		if err != nil {
 			p.logger.Error().Err(err).Msg("failed to move the changed resource in the index")
 		}
-		return
+		p.reindexSpace(ev, e.Ref, e.Executant, e.SpaceOwner)
 	case events.ContainerCreated:
-		ref = e.Ref
-		owner = &user.User{
-			Id: e.Executant,
-		}
+		p.reindexSpace(ev, e.Ref, e.Executant, e.SpaceOwner)
 	case events.FileUploaded:
-		ref = e.Ref
-		owner = &user.User{
-			Id: e.Executant,
-		}
+		p.reindexSpace(ev, e.Ref, e.Executant, e.SpaceOwner)
 	case events.FileTouched:
-		ref = e.Ref
-		owner = &user.User{
-			Id: e.Executant,
-		}
+		p.reindexSpace(ev, e.Ref, e.Executant, e.SpaceOwner)
 	case events.FileVersionRestored:
-		ref = e.Ref
-		owner = &user.User{
-			Id: e.Executant,
-		}
+		p.reindexSpace(ev, e.Ref, e.Executant, e.SpaceOwner)
 	default:
 		// Not sure what to do here. Skip.
 		return
 	}
-	p.logger.Debug().Interface("event", ev).Msg("resource has been changed, updating the document")
-
-	ownerCtx, err := p.getAuthContext(owner)
-	if err != nil {
-		return
-	}
+}
 
-	statRes, err := p.statResource(ownerCtx, ref, owner)
-	if err != nil {
-		p.logger.Error().Err(err).
-			Str("storageid", ref.GetResourceId().GetStorageId()).
-			Str("spaceid", ref.GetResourceId().GetSpaceId()).
-			Str("opaqueid", ref.GetResourceId().GetOpaqueId()).
-			Str("path", ref.GetPath()).
-			Msg("failed to make stat call for changed resource")
-		return
-	}
-	if statRes.Status.Code != rpc.Code_CODE_OK {
-		p.logger.Error().Interface("statRes", statRes).
-			Str("storageid", ref.GetResourceId().GetStorageId()).
-			Str("spaceid", ref.GetResourceId().GetSpaceId()).
-			Str("opaqueid", ref.GetResourceId().GetOpaqueId()).
-			Str("path", ref.GetPath()).
-			Msg("failed to stat the changed resource")
-		return
-	}
+func (p *Provider) reindexSpace(ev interface{}, ref *provider.Reference, executant, owner *user.UserId) {
+	p.logger.Debug().Interface("event", ev).Msg("resource has been changed, scheduling a space resync")
 
-	ref, err = p.resolveReference(ownerCtx, ref, statRes.Info)
-	if err != nil {
-		p.logger.Error().Err(err).Msg("error resolving reference")
-		return
+	spaceID := &provider.StorageSpaceId{
+		OpaqueId: storagespace.FormatResourceID(provider.ResourceId{
+			StorageId: ref.GetResourceId().GetStorageId(),
+			SpaceId:   ref.GetResourceId().GetSpaceId(),
+		}),
 	}
-	err = p.indexClient.Add(ref, statRes.Info)
-	if err != nil {
-		p.logger.Error().Err(err).Msg("error adding updating the resource in the index")
+	if owner != nil {
+		p.indexSpaceDebouncer.Debounce(spaceID, owner)
 	} else {
-		p.logDocCount()
+		p.indexSpaceDebouncer.Debounce(spaceID, executant)
 	}
 }