Add support for scopes and collections (#2)

Motivation
------------
Allow usage with Couchbase Server 7.0 scopes and collections.

Modifications
---------------
Upgrade to couchbase-index-manager 2.0.0-beta001.

Add scope name and collection name to the CRD for indexes.

Change index matching to use scope, collection, and name instead of just
the name.

Results
--------
Indexes may be managed on non-default collections.

.devcontainer/devcontainer.json

@@ -31,6 +31,7 @@
 
  // Add the IDs of extensions you want installed when the container is created.
  "extensions": [
+ "github.vscode-pull-request-github",
  "golang.Go",
  "ms-azuretools.vscode-docker",
  "ms-kubernetes-tools.vscode-kubernetes-tools"

api/v1beta1/couchbaseindexset_types.go

@@ -45,6 +45,14 @@ type GlobalSecondaryIndex struct {
  //+kubebuilder:validation:Pattern:=^[A-Za-z][A-Za-z0-9#_]*$
  // Name of the index
  Name string `json:"name"`
+ //+kubebuilder:validation:MinLength:=1
+ //+kubebuilder:validation:Pattern:="^_default$|^[A-Za-z0-9\\-][A-Za-z0-9_\\-%]*$"
+ // Name of the index's scope, assumes "_default" if not present
+ ScopeName *string `json:"scopeName,omitempty"`
+ //+kubebuilder:validation:MinLength:=1
+ //+kubebuilder:validation:Pattern:="^_default$|^[A-Za-z0-9\\-][A-Za-z0-9_\\-%]*$"
+ // Name of the index's collection, assumes "_default" if not present
+ CollectionName *string `json:"collectionName,omitempty"`
  //+kubebuilder:validation:MinItems:=1
  // List of properties or deterministic functions which make up the index key
  IndexKey []string `json:"indexKey"`

cbim/identifiers.go

@@ -0,0 +1,77 @@
+package cbim
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+
+ couchbasev1beta1 "github.com/brantburnett/couchbase-index-operator/api/v1beta1"
+)
+
+const (
+ defaultScopeName = "_default"
+ defaultCollectionName = "_default"
+)
+
+// Uniquely defines a global secondary index.
+type GlobalSecondaryIndexIdentifier struct {
+ // Name of the index
+ Name string
+ // Name of the index's scope
+ ScopeName string
+ // Name of the index's collection
+ CollectionName string
+}
+
+func defaultedName(name *string) string {
+ if name == nil {
+ return defaultScopeName
+ }
+
+ return *name
+}
+
+func GetIndexIdentifier(index couchbasev1beta1.GlobalSecondaryIndex) GlobalSecondaryIndexIdentifier {
+ return GlobalSecondaryIndexIdentifier{
+ Name: index.Name,
+ ScopeName: defaultedName(index.ScopeName),
+ CollectionName: defaultedName(index.CollectionName),
+ }
+}
+
+func (identifier GlobalSecondaryIndexIdentifier) IsDefaultCollection() bool {
+ return identifier.ScopeName == defaultScopeName && identifier.CollectionName == defaultCollectionName
+}
+
+func (identifier GlobalSecondaryIndexIdentifier) ToString() string {
+ if identifier.IsDefaultCollection() {
+ return identifier.Name
+ }
+
+ return fmt.Sprintf("%s.%s.%s", identifier.ScopeName, identifier.CollectionName, identifier.Name)
+}
+
+func ParseIndexIdentifierString(identifier string) (GlobalSecondaryIndexIdentifier, error) {
+ if identifier == "" {
+ return GlobalSecondaryIndexIdentifier{}, errors.New("invalid index identifier")
+ }
+
+ split := strings.Split(identifier, ".")
+ if len(split) == 1 {
+ return GlobalSecondaryIndexIdentifier{
+ ScopeName: defaultScopeName,
+ CollectionName: defaultCollectionName,
+ Name: split[0],
+ }, nil
+ }
+
+ if len(split) != 3 || split[0] == "" || split[1] == "" || split[2] == "" {
+ return GlobalSecondaryIndexIdentifier{}, errors.New("invalid index identifier")
+ }
+
+ return GlobalSecondaryIndexIdentifier{
+ ScopeName: split[0],
+ CollectionName: split[1],
+ Name: split[2],
+ }, nil
+}

cbim/spec.go

@@ -25,31 +25,33 @@ import (
  couchbasev1beta1 "github.com/brantburnett/couchbase-index-operator/api/v1beta1"
 )
 
-func GenerateYaml(indexSet *couchbasev1beta1.CouchbaseIndexSet, deletingIndexNames *[]string) (string, error) {
+func GenerateYaml(indexSet *couchbasev1beta1.CouchbaseIndexSet, deletingIndexes *[]GlobalSecondaryIndexIdentifier) (string, error) {
  var sb strings.Builder
 
- definedIndexNames := map[string]bool{}
+ definedIndexes := map[GlobalSecondaryIndexIdentifier]bool{}
 
  if indexSet.GetDeletionTimestamp() == nil {
  // Only create indices if we're not deleting the index set
  // If we are deleting, this will leave definedIndexNames empty so all indices are deleted
 
  for _, gsi := range indexSet.Spec.Indices {
- definedIndexNames[gsi.Name] = true
+ definedIndexes[GetIndexIdentifier(gsi)] = true
 
  if err := addIndexSpec(&sb, createIndexSpec(&gsi)); err != nil {
  return "", err
  }
  }
  }
 
- *deletingIndexNames = []string{}
- for _, indexName := range indexSet.Status.Indices {
- if !definedIndexNames[indexName] {
- *deletingIndexNames = append(*deletingIndexNames, indexName)
+ *deletingIndexes = []GlobalSecondaryIndexIdentifier{}
+ for _, index := range indexSet.Status.Indices {
+ if indexIdentifier, err := ParseIndexIdentifierString(index); err == nil {
+ if !definedIndexes[indexIdentifier] {
+ *deletingIndexes = append(*deletingIndexes, indexIdentifier)
 
- if err := addIndexSpec(&sb, createIndexDeleteSpec(indexName)); err != nil {
- return "", err
+ if err := addIndexSpec(&sb, createIndexDeleteSpec(indexIdentifier)); err != nil {
+ return "", err
+ }
  }
  }
  }
@@ -74,6 +76,8 @@ func addIndexSpec(sb *strings.Builder, spec IndexSpec) error {
 func createIndexSpec(gsi *couchbasev1beta1.GlobalSecondaryIndex) IndexSpec {
  return IndexSpec{
  Name: gsi.Name,
+ Scope: gsi.ScopeName,
+ Collection: gsi.CollectionName,
  IndexKey: &gsi.IndexKey,
  Condition: gsi.Condition,
  NumReplicas: gsi.NumReplicas,
@@ -103,9 +107,11 @@ func mapPartitionStrategy(strategy *string) *string {
  return &result
 }
 
-func createIndexDeleteSpec(indexName string) IndexSpec {
+func createIndexDeleteSpec(indexIdentifier GlobalSecondaryIndexIdentifier) IndexSpec {
  return IndexSpec{
- Name: indexName,
+ Name: indexIdentifier.Name,
+ Scope: &indexIdentifier.ScopeName,
+ Collection: &indexIdentifier.CollectionName,
  Lifecycle: &LifecycleSpec{
  Drop: pointer.BoolPtr(true),
  },

cbim/suite_test.go

@@ -0,0 +1,133 @@
+package cbim
+
+import (
+ "testing"
+
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+ "sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+ //+kubebuilder:scaffold:imports
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+func TestAPIs(t *testing.T) {
+ RegisterFailHandler(Fail)
+
+ RunSpecsWithDefaultAndCustomReporters(t,
+ "V1Beta1 Suite",
+ []Reporter{printer.NewlineReporter{}})
+}
+
+var _ = Describe("GlobalSecondaryIndexIdentifier.ToString", func() {
+
+ It("should exclude the default collection name", func() {
+ // Arrange
+
+ identifier := GlobalSecondaryIndexIdentifier{
+ ScopeName: "_default",
+ CollectionName: "_default",
+ Name: "my_index",
+ }
+
+ // Act
+
+ result := identifier.ToString()
+
+ // Assert
+
+ Expect(result).To(Equal("my_index"))
+ })
+
+ It("should return a dotted name", func() {
+ // Arrange
+
+ identifier := GlobalSecondaryIndexIdentifier{
+ ScopeName: "scope",
+ CollectionName: "_default",
+ Name: "my_index",
+ }
+
+ // Act
+
+ result := identifier.ToString()
+
+ // Assert
+
+ Expect(result).To(Equal("scope._default.my_index"))
+ })
+})
+
+var _ = Describe("ParseIndexIdentifierString", func() {
+
+ It("should error on empty string", func() {
+ // Act
+
+ _, err := ParseIndexIdentifierString("")
+
+ // Assert
+
+ Expect(err).NotTo(BeNil())
+ })
+
+ It("should error on empty segment", func() {
+ // Act
+
+ _, err := ParseIndexIdentifierString("scope..name")
+
+ // Assert
+
+ Expect(err).NotTo(BeNil())
+ })
+
+ It("should error on two segments", func() {
+ // Act
+
+ _, err := ParseIndexIdentifierString("scope.name")
+
+ // Assert
+
+ Expect(err).NotTo(BeNil())
+ })
+
+ It("should error on four segments", func() {
+ // Act
+
+ _, err := ParseIndexIdentifierString("scope.collection.name.extra")
+
+ // Assert
+
+ Expect(err).NotTo(BeNil())
+ })
+
+ It("should return simple name in default collection", func() {
+ // Act
+
+ result, err := ParseIndexIdentifierString("name")
+
+ // Assert
+
+ Expect(result).To(Equal(GlobalSecondaryIndexIdentifier{
+ ScopeName: "_default",
+ CollectionName: "_default",
+ Name: "name",
+ }))
+ Expect(err).To(BeNil())
+ })
+
+ It("should return dotted name", func() {
+ // Act
+
+ result, err := ParseIndexIdentifierString("scope.collection.name")
+
+ // Assert
+
+ Expect(result).To(Equal(GlobalSecondaryIndexIdentifier{
+ ScopeName: "scope",
+ CollectionName: "collection",
+ Name: "name",
+ }))
+ Expect(err).To(BeNil())
+ })
+})

cbim/types.go

@@ -33,6 +33,8 @@ type LifecycleSpec struct {
 type IndexSpec struct {
  Type string `json:"type,omitempty"`
  Name string `json:"name"`
+ Scope *string `json:"scope,omitempty"`
+ Collection *string `json:"collection,omitempty"`
  IsPrimary *bool `json:"is_primary,omitempty"`
  IndexKey *[]string `json:"index_key,omitempty"`
  Condition *string `json:"condition,omitempty"`

config/crd/bases/couchbase.btburnett.com_couchbaseindexsets.yaml

@@ -109,6 +109,12 @@ spec:
  description: Defines the desired state of a Couchbase Global Secondary
  Index
  properties:
+ collectionName:
+ description: Name of the index's collection, assumes "_default"
+ if not present
+ minLength: 1
+ pattern: ^_default$|^[A-Za-z0-9\-][A-Za-z0-9_\-%]*$
+ type: string
  condition:
  description: Conditions to filter documents included on the
  index
@@ -157,6 +163,12 @@ spec:
  description: Enable for Sync Gateway indices to preserve deleted
  XAttrs
  type: boolean
+ scopeName:
+ description: Name of the index's scope, assumes "_default" if
+ not present
+ minLength: 1
+ pattern: ^_default$|^[A-Za-z0-9\-][A-Za-z0-9_\-%]*$
+ type: string
  required:
  - indexKey
  - name

config/samples/couchbase_v1beta1_couchbaseindexset.yaml

@@ -13,6 +13,8 @@ spec:
  - type
  - id
  - name: example2
+ scopeName: my_scope
+ collectionName: my_collection
  indexKey:
  - type
  - name

controllers/couchbaseindexset_configmap.go

@@ -52,7 +52,7 @@ func (context *CouchbaseIndexSetReconcileContext) reconcileConfigMap() (string,
  }
  }
 
- yaml, err := cbim.GenerateYaml(&context.IndexSet, &context.DeletingIndexNames)
+ yaml, err := cbim.GenerateYaml(&context.IndexSet, &context.DeletingIndexes)
  if err != nil {
  context.Error(err, "Error generating index spec")
  return "", err

controllers/couchbaseindexset_controller.go

@@ -34,6 +34,7 @@ import (
  "sigs.k8s.io/controller-runtime/pkg/predicate"
 
  v1beta1 "github.com/brantburnett/couchbase-index-operator/api/v1beta1"
+ cbim "github.com/brantburnett/couchbase-index-operator/cbim"
  "github.com/go-logr/logr"
 )
 
@@ -53,11 +54,11 @@ type CouchbaseIndexSetReconcileContext struct {
  logr.Logger
  Reconciler *CouchbaseIndexSetReconciler
 
- IndexSet  v1beta1.CouchbaseIndexSet
- ConnectionString  string
- AdminSecretName  string
- DeletingIndexNames []string
- IsDeleting  bool
+ IndexSet v1beta1.CouchbaseIndexSet
+ ConnectionString string
+ AdminSecretName string
+ DeletingIndexes []cbim.GlobalSecondaryIndexIdentifier
+ IsDeleting bool
 }
 
 //+kubebuilder:rbac:groups=couchbase.btburnett.com,namespace=system,resources=couchbaseindexsets,verbs=get;list;watch;create;update;patch;delete