add support for intersection & union in search operations (#968)

* add support for intersection and union search

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* try testing operations

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* don't regenerate swagger

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* fix e2e test

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* use right validation tag

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* re-implement collection, test with minisign

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* remove interface

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* comments

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

* organize test

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

Signed-off-by: Samsondeen Dare <samsondeen.dare@hashicorp.com>

cmd/rekor-cli/app/pflags.go

@@ -38,6 +38,7 @@ const (
 	uuidFlag      FlagType = "uuid"
 	shaFlag       FlagType = "sha"
 	emailFlag     FlagType = "email"
+	operatorFlag  FlagType = "operator"
 	logIndexFlag  FlagType = "logIndex"
 	pkiFormatFlag FlagType = "pkiFormat"
 	typeFlag      FlagType = "type"
@@ -67,6 +68,10 @@ func initializePFlagMap() {
 			// this validates a valid sha256 checksum which is optionally prefixed with 'sha256:'
 			return valueFactory(shaFlag, validateSHAValue, "")
 		},
+		operatorFlag: func() pflag.Value {
+			// this validates a valid operator name
+			return valueFactory(shaFlag, validateString("oneof=and or"), "")
+		},
 		emailFlag: func() pflag.Value {
 			// this validates an email address
 			return valueFactory(emailFlag, validateString("required,email"), "")

cmd/rekor-cli/app/search.go

@@ -58,6 +58,8 @@ func addSearchPFlags(cmd *cobra.Command) error {
 	cmd.Flags().Var(NewFlagValue(shaFlag, ""), "sha", "the SHA256 or SHA1 sum of the artifact")
 
 	cmd.Flags().Var(NewFlagValue(emailFlag, ""), "email", "email associated with the public key's subject")
+
+	cmd.Flags().Var(NewFlagValue(operatorFlag, ""), "operator", "operator to use for the search. supported values are 'and' and 'or'")
 	return nil
 }
 
@@ -142,6 +144,8 @@ var searchCmd = &cobra.Command{
 			params.Query.Hash = "sha256:" + hashVal
 		}
 
+		params.Query.Operator = viper.GetString("operator")
+
 		publicKeyStr := viper.GetString("public-key")
 		if publicKeyStr != "" {
 			params.Query.PublicKey = &models.SearchIndexPublicKey{}

openapi.yaml

@@ -510,6 +510,9 @@ definitions:
       hash:
         type: string
         pattern: '^(sha256:)?[0-9a-fA-F]{64}$|^(sha1:)?[0-9a-fA-F]{40}$'
+      operator:
+        type: string
+        enum: ['and','or']
 
   SearchLogQuery:
     type: object

pkg/api/index.go

@@ -35,15 +35,21 @@ import (
 func SearchIndexHandler(params index.SearchIndexParams) middleware.Responder {
 	httpReqCtx := params.HTTPRequest.Context()
 
-	var result []string
+	queryOperator := params.Query.Operator
+	// default to "or" if no operator is specified
+	if params.Query.Operator == "" {
+		queryOperator = "or"
+	}
+	var result = NewCollection(queryOperator)
+
 	if params.Query.Hash != "" {
 		// This must be a valid sha256 hash
 		sha := util.PrefixSHA(params.Query.Hash)
 		var resultUUIDs []string
 		if err := redisClient.Do(httpReqCtx, radix.Cmd(&resultUUIDs, "LRANGE", strings.ToLower(sha), "0", "-1")); err != nil {
 			return handleRekorAPIError(params, http.StatusInternalServerError, err, redisUnexpectedResult)
 		}
-		result = append(result, resultUUIDs...)
+		result.Add(resultUUIDs)
 	}
 	if params.Query.PublicKey != nil {
 		af, err := pki.NewArtifactFactory(pki.Format(swag.StringValue(params.Query.PublicKey.Format)))
@@ -70,17 +76,17 @@ func SearchIndexHandler(params index.SearchIndexParams) middleware.Responder {
 		if err := redisClient.Do(httpReqCtx, radix.Cmd(&resultUUIDs, "LRANGE", strings.ToLower(hex.EncodeToString(keyHash[:])), "0", "-1")); err != nil {
 			return handleRekorAPIError(params, http.StatusInternalServerError, err, redisUnexpectedResult)
 		}
-		result = append(result, resultUUIDs...)
+		result.Add(resultUUIDs)
 	}
 	if params.Query.Email != "" {
 		var resultUUIDs []string
 		if err := redisClient.Do(httpReqCtx, radix.Cmd(&resultUUIDs, "LRANGE", strings.ToLower(params.Query.Email.String()), "0", "-1")); err != nil {
 			return handleRekorAPIError(params, http.StatusInternalServerError, err, redisUnexpectedResult)
 		}
-		result = append(result, resultUUIDs...)
+		result.Add(resultUUIDs)
 	}
 
-	return index.NewSearchIndexOK().WithPayload(result)
+	return index.NewSearchIndexOK().WithPayload(result.Values())
 }
 
 func SearchIndexNotImplementedHandler(params index.SearchIndexParams) middleware.Responder {
@@ -100,3 +106,86 @@ func addToIndex(ctx context.Context, key, value string) error {
 func storeAttestation(ctx context.Context, uuid string, attestation []byte) error {
 	return storageClient.StoreAttestation(ctx, uuid, attestation)
 }
+
+// Uniq is a collection of unique elements.
+type Uniq map[string]struct{}
+
+func NewUniq() Uniq {
+	return make(Uniq)
+}
+
+func (u Uniq) Add(elements ...string) {
+	for _, e := range elements {
+		u[e] = struct{}{}
+	}
+}
+
+func (u Uniq) Values() []string {
+	var result []string
+	for k := range u {
+		result = append(result, k)
+	}
+	return result
+}
+
+// Intersect returns the intersection of two collections.
+func (u Uniq) Intersect(other Uniq) Uniq {
+	result := make(Uniq)
+	for k := range u {
+		if _, ok := other[k]; ok {
+			result[k] = struct{}{}
+		}
+	}
+	return result
+}
+
+// Union returns the union of two collections.
+func (u Uniq) Union(other Uniq) Uniq {
+	result := make(Uniq)
+	for k := range u {
+		result[k] = struct{}{}
+	}
+	for k := range other {
+		result[k] = struct{}{}
+	}
+	return result
+}
+
+// Collection is a collection of sets.
+//
+// its resulting values is a union or intersection of all the sets, depending on the operator.
+type Collection struct {
+	subsets  []Uniq
+	operator string
+}
+
+// NewCollection creates a new collection.
+func NewCollection(operator string) *Collection {
+	return &Collection{
+		subsets:  []Uniq{},
+		operator: operator,
+	}
+}
+
+// Add adds the elements into a new subset in the collection.
+func (u *Collection) Add(elements []string) {
+	subset := Uniq{}
+	subset.Add(elements...)
+	u.subsets = append(u.subsets, subset)
+}
+
+// Values flattens the subsets using the operator, and returns the collection as a slice of strings.
+func (u *Collection) Values() []string {
+	if len(u.subsets) == 0 {
+		return []string{}
+	}
+	subset := u.subsets[0]
+	for i := 1; i < len(u.subsets); i++ {
+		if strings.EqualFold(u.operator, "and") {
+			subset = subset.Intersect(u.subsets[i])
+		} else {
+			subset = subset.Union(u.subsets[i])
+		}
+	}
+	return subset.Values()
+}

pkg/api/index_test.go

@@ -0,0 +1,85 @@
+//
+// Copyright 2021 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+func Test_Collection(t *testing.T) {
+
+	vals := []string{"foo", "bar", "baz", "baz", "baz"}
+
+	t.Run("Unique", func(t *testing.T) {
+		unq := NewUniq()
+		unq.Add(vals...)
+
+		if len(unq.Values()) != 3 {
+			t.Errorf("expected 3 unique values, got %d", len(unq.Values()))
+		}
+		expected := []string{"foo", "bar", "baz"}
+		if !testEqualNoOrder(t, expected, unq.Values()) {
+			t.Errorf("expected %v, got %v", expected, unq.Values())
+		}
+	})
+
+	t.Run("Collection", func(t *testing.T) {
+
+		uniq1 := []string{"foo", "bar", "baz"}
+		uniq2 := []string{"foo", "bar", "baz"}
+		uniq3 := []string{"corge", "grault", "garply", "foo"}
+
+		tests := []struct {
+			name     string
+			operator string
+			expected []string
+		}{
+			{name: "with 'and' operator",
+				operator: "and",
+				expected: []string{"foo"},
+			},
+			{name: "with 'or' operator",
+				operator: "or",
+				expected: []string{"foo", "bar", "baz", "corge", "grault", "garply"},
+			},
+		}
+
+		for _, test := range tests {
+			t.Run(test.name, func(t *testing.T) {
+				c := NewCollection(test.operator)
+				c.Add(uniq1)
+				c.Add(uniq2)
+				c.Add(uniq3)
+
+				if !testEqualNoOrder(t, test.expected, c.Values()) {
+					t.Errorf("expected %v, got %v", test.expected, c.Values())
+				}
+			})
+		}
+
+	})
+
+}
+
+// testEqualNoOrder compares two slices of strings without considering order.
+func testEqualNoOrder(t *testing.T, expected, actual []string) bool {
+	t.Helper()
+	less := func(a, b string) bool { return a < b }
+	return cmp.Diff(actual, expected, cmpopts.SortSlices(less)) == ""
+}