Search: Return the number of total matches

Makefile

@@ -181,7 +181,7 @@ go-coverage:
 
 .PHONY: protobuf
 protobuf:
- @for mod in $(OCIS_MODULES); do \
+ @for mod in ./services/thumbnails ./services/store ./services/settings; do \
  echo -n "% protobuf $$mod: "; $(MAKE) --no-print-directory -C $$mod protobuf || exit 1; \
  done
 

changelog/unreleased/add-total-to-search-results.md

@@ -0,0 +1,6 @@
+Enhancement: Add number of total matches to the search result
+
+The search service now returns the number of total matches alongside the
+results.
+
+https://github.com/owncloud/ocis/issues/4189

protogen/gen/ocis/services/search/v0/search.swagger.json

@@ -280,6 +280,10 @@
  "nextPageToken": {
  "type": "string",
  "title": "Token to retrieve the next page of results, or empty if there are no\nmore results in the list"
+ },
+ "totalMatches": {
+ "type": "integer",
+ "format": "int32"
  }
  }
  },
@@ -312,6 +316,10 @@
  "nextPageToken": {
  "type": "string",
  "title": "Token to retrieve the next page of results, or empty if there are no\nmore results in the list"
+ },
+ "totalMatches": {
+ "type": "integer",
+ "format": "int32"
  }
  }
  }

protogen/proto/ocis/services/search/v0/search.proto

@@ -76,6 +76,7 @@ message SearchResponse {
  // Token to retrieve the next page of results, or empty if there are no
  // more results in the list
  string next_page_token = 2;
+ int32 total_matches = 3;
 }
 
 message SearchIndexRequest {
@@ -91,11 +92,12 @@ message SearchIndexRequest {
 }
 
 message SearchIndexResponse {
-  repeated ocis.messages.search.v0.Match matches = 1;
+ repeated ocis.messages.search.v0.Match matches = 1;
 
- // Token to retrieve the next page of results, or empty if there are no
- // more results in the list
- string next_page_token = 2;
+ // Token to retrieve the next page of results, or empty if there are no
+ // more results in the list
+ string next_page_token = 2;
+ int32 total_matches = 3;
 }
 
 message IndexSpaceRequest {

services/search/pkg/search/index/index.go

@@ -27,12 +27,13 @@ import (
  "strings"
  "time"
 
- "github.com/blevesearch/bleve/v2"
+ bleve "github.com/blevesearch/bleve/v2"
  "github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
  "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
  "github.com/blevesearch/bleve/v2/analysis/token/lowercase"
  "github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
  "github.com/blevesearch/bleve/v2/mapping"
+ "github.com/blevesearch/bleve/v2/search"
  "github.com/cs3org/reva/v2/pkg/storagespace"
  "google.golang.org/protobuf/types/known/timestamppb"
 
@@ -239,15 +240,16 @@ func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (
 
  matches := []*searchmsg.Match{}
  for _, h := range res.Hits {
- match, err := fromFields(h.Fields)
+ match, err := fromDocumentMatch(h)
  if err != nil {
  return nil, err
  }
  matches = append(matches, match)
  }
 
  return &searchsvc.SearchIndexResponse{
- Matches: matches,
+ Matches: matches,
+ TotalMatches: int32(res.Total),
  }, nil
 }
 
@@ -311,32 +313,33 @@ func fieldsToEntity(fields map[string]interface{}) *indexDocument {
  return doc
 }
 
-func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) {
- rootID, err := storagespace.ParseID(fields["RootID"].(string))
+func fromDocumentMatch(hit *search.DocumentMatch) (*searchmsg.Match, error) {
+ rootID, err := storagespace.ParseID(hit.Fields["RootID"].(string))
  if err != nil {
  return nil, err
  }
- rID, err := storagespace.ParseID(fields["ID"].(string))
+ rID, err := storagespace.ParseID(hit.Fields["ID"].(string))
  if err != nil {
  return nil, err
  }
 
  match := &searchmsg.Match{
+ Score: float32(hit.Score),
  Entity: &searchmsg.Entity{
  Ref: &searchmsg.Reference{
  ResourceId: resourceIDtoSearchID(rootID),
- Path: fields["Path"].(string),
+ Path: hit.Fields["Path"].(string),
  },
  Id: resourceIDtoSearchID(rID),
- Name: fields["Name"].(string),
- Size: uint64(fields["Size"].(float64)),
- Type: uint64(fields["Type"].(float64)),
- MimeType: fields["MimeType"].(string),
- Deleted: fields["Deleted"].(bool),
+ Name: hit.Fields["Name"].(string),
+ Size: uint64(hit.Fields["Size"].(float64)),
+ Type: uint64(hit.Fields["Type"].(float64)),
+ MimeType: hit.Fields["MimeType"].(string),
+ Deleted: hit.Fields["Deleted"].(bool),
  },
  }
 
- if mtime, err := time.Parse(time.RFC3339, fields["Mtime"].(string)); err == nil {
+ if mtime, err := time.Parse(time.RFC3339, hit.Fields["Mtime"].(string)); err == nil {
  match.Entity.LastModifiedTime = &timestamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
  }
 

services/search/pkg/search/index/index_test.go

@@ -190,6 +190,19 @@ var _ = Describe("Index", func() {
  assertDocCount(ref.ResourceId, "Name:*"+ref.ResourceId.OpaqueId+"*", 0)
  })
 
+ It("returns the total number of hits", func() {
+ res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
+ Query: "Name:foo.pdf",
+ Ref: &searchmsg.Reference{
+ ResourceId: &searchmsg.ResourceID{
+ StorageId: ref.ResourceId.StorageId,
+ OpaqueId: ref.ResourceId.OpaqueId,
+ },
+ },
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(res.TotalMatches).To(Equal(int32(1)))
+ })
  It("returns all desired fields", func() {
  matches := assertDocCount(ref.ResourceId, "Name:foo.pdf", 1)
  match := matches[0]
@@ -201,6 +214,7 @@ var _ = Describe("Index", func() {
  Expect(match.Entity.Type).To(Equal(uint64(ri.Type)))
  Expect(match.Entity.MimeType).To(Equal(ri.MimeType))
  Expect(match.Entity.Deleted).To(BeFalse())
+ Expect(match.Score > 0).To(BeTrue())
  Expect(uint64(match.Entity.LastModifiedTime.AsTime().Unix())).To(Equal(ri.Mtime.Seconds))
  })
 

services/search/pkg/search/provider/searchprovider.go

@@ -4,6 +4,7 @@ import (
  "context"
  "fmt"
  "path/filepath"
+ "sort"
  "strings"
  "time"
 
@@ -44,6 +45,18 @@ type Provider struct {
  machineAuthAPIKey string
 }
 
+type MatchArray []*searchmsg.Match
+
+func (s MatchArray) Len() int {
+ return len(s)
+}
+func (s MatchArray) Swap(i, j int) {
+ s[i], s[j] = s[j], s[i]
+}
+func (s MatchArray) Less(i, j int) bool {
+ return s[i].Score > s[j].Score
+}
+
 func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, machineAuthAPIKey string, eventsChan <-chan interface{}, logger log.Logger) *Provider {
  p := &Provider{
  gwClient: gwClient,
@@ -98,7 +111,8 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
  mountpointMap[grantSpaceId] = space.Id.OpaqueId
  }
 
- matches := []*searchmsg.Match{}
+ matches := MatchArray{}
+ total := int32(0)
  for _, space := range listSpacesRes.StorageSpaces {
  var mountpointRootId *searchmsg.ResourceID
  mountpointPrefix := ""
@@ -154,6 +168,7 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
  }
  p.logger.Debug().Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done")
 
+ total += res.TotalMatches
  for _, match := range res.Matches {
  if mountpointPrefix != "" {
  match.Entity.Ref.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Entity.Ref.Path, mountpointPrefix))
@@ -165,8 +180,19 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
  }
  }
 
+ // compile one sorted list of matches from all spaces and apply the limit if needed
+ sort.Sort(matches)
+ limit := req.PageSize
+ if limit == 0 {
+ limit = 200
+ }
+ if int32(len(matches)) > limit {
+ matches = matches[0:limit]
+ }
+
  return &searchsvc.SearchResponse{
- Matches: matches,
+ Matches: matches,
+ TotalMatches: total,
  }, nil
 }