diff --git a/docs/Manual/Deployment/Kubernetes/DeploymentResource.md b/docs/Manual/Deployment/Kubernetes/DeploymentResource.md index df06aa86a..27e1383ee 100644 --- a/docs/Manual/Deployment/Kubernetes/DeploymentResource.md +++ b/docs/Manual/Deployment/Kubernetes/DeploymentResource.md @@ -205,17 +205,6 @@ replication in the cluster. When enabled, the cluster will contain a number of `syncmaster` & `syncworker` servers. The default value is `false`. -### `spec.sync.image: string` - -This setting specifies the docker image to use for all ArangoSync servers. -When not specified, the `spec.image` value is used. - -### `spec.sync.imagePullPolicy: string` - -This setting specifies the pull policy for the docker image to use for all ArangoSync servers. -For possible values, see `spec.imagePullPolicy`. -When not specified, the `spec.imagePullPolicy` value is used. - ### `spec.sync.externalAccess.type: string` This setting specifies the type of `Service` that will be created to provide diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index fea5ef888..6c4ccaa8e 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -144,7 +144,7 @@ func (s *DeploymentSpec) SetDefaults(deploymentName string) { s.RocksDB.SetDefaults() s.Authentication.SetDefaults(deploymentName + "-jwt") s.TLS.SetDefaults(deploymentName + "-ca") - s.Sync.SetDefaults(s.GetImage(), s.GetImagePullPolicy(), deploymentName+"-sync-jwt", deploymentName+"-sync-client-auth-ca", deploymentName+"-sync-ca") + s.Sync.SetDefaults(deploymentName+"-sync-jwt", deploymentName+"-sync-client-auth-ca", deploymentName+"-sync-ca", deploymentName+"-sync-mt") s.Single.SetDefaults(ServerGroupSingle, s.GetMode().HasSingleServers(), s.GetMode()) s.Agents.SetDefaults(ServerGroupAgents, s.GetMode().HasAgents(), s.GetMode()) s.DBServers.SetDefaults(ServerGroupDBServers, s.GetMode().HasDBServers(), s.GetMode()) diff --git a/pkg/apis/deployment/v1alpha/monitoring_spec.go b/pkg/apis/deployment/v1alpha/monitoring_spec.go index 719468cd7..5c11b590e 100644 --- a/pkg/apis/deployment/v1alpha/monitoring_spec.go +++ b/pkg/apis/deployment/v1alpha/monitoring_spec.go @@ -46,8 +46,12 @@ func (s MonitoringSpec) Validate() error { } // SetDefaults fills in missing defaults -func (s *MonitoringSpec) SetDefaults() { - // Nothing needed +func (s *MonitoringSpec) SetDefaults(defaultTokenSecretName string) { + if s.GetTokenSecretName() == "" { + // Note that we don't check for nil here, since even a specified, but empty + // string should result in the default value. + s.TokenSecretName = util.NewString(defaultTokenSecretName) + } } // SetDefaultsFrom fills unspecified fields with a value from given source spec. diff --git a/pkg/apis/deployment/v1alpha/monitoring_spec_test.go b/pkg/apis/deployment/v1alpha/monitoring_spec_test.go index 8ece949d0..855f74252 100644 --- a/pkg/apis/deployment/v1alpha/monitoring_spec_test.go +++ b/pkg/apis/deployment/v1alpha/monitoring_spec_test.go @@ -42,10 +42,15 @@ func TestMonitoringSpecValidate(t *testing.T) { func TestMonitoringSpecSetDefaults(t *testing.T) { def := func(spec MonitoringSpec) MonitoringSpec { - spec.SetDefaults() + spec.SetDefaults("") + return spec + } + def2 := func(spec MonitoringSpec) MonitoringSpec { + spec.SetDefaults("def2") return spec } assert.Equal(t, "", def(MonitoringSpec{}).GetTokenSecretName()) + assert.Equal(t, "def2", def2(MonitoringSpec{}).GetTokenSecretName()) assert.Equal(t, "foo", def(MonitoringSpec{TokenSecretName: util.NewString("foo")}).GetTokenSecretName()) } diff --git a/pkg/apis/deployment/v1alpha/server_group_spec.go b/pkg/apis/deployment/v1alpha/server_group_spec.go index 5914aee8e..df267baee 100644 --- a/pkg/apis/deployment/v1alpha/server_group_spec.go +++ b/pkg/apis/deployment/v1alpha/server_group_spec.go @@ -105,6 +105,8 @@ func (s *ServerGroupSpec) SetDefaults(group ServerGroup, used bool, mode Deploym default: s.Count = util.NewInt(3) } + } else if s.GetCount() > 0 && !used { + s.Count = util.NewInt(0) } if _, found := s.Resources.Requests[v1.ResourceStorage]; !found { switch group { diff --git a/pkg/apis/deployment/v1alpha/sync_spec.go b/pkg/apis/deployment/v1alpha/sync_spec.go index 25d7f889f..3b0473e2f 100644 --- a/pkg/apis/deployment/v1alpha/sync_spec.go +++ b/pkg/apis/deployment/v1alpha/sync_spec.go @@ -24,16 +24,13 @@ package v1alpha import ( "github.com/pkg/errors" - "k8s.io/api/core/v1" "github.com/arangodb/kube-arangodb/pkg/util" ) // SyncSpec holds dc2dc replication specific configuration settings type SyncSpec struct { - Enabled *bool `json:"enabled,omitempty"` - Image *string `json:"image,omitempty"` - ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` + Enabled *bool `json:"enabled,omitempty"` ExternalAccess SyncExternalAccessSpec `json:"externalAccess"` Authentication SyncAuthenticationSpec `json:"auth"` @@ -46,24 +43,11 @@ func (s SyncSpec) IsEnabled() bool { return util.BoolOrDefault(s.Enabled) } -// GetImage returns the value of image. -func (s SyncSpec) GetImage() string { - return util.StringOrDefault(s.Image) -} - -// GetImagePullPolicy returns the value of imagePullPolicy. -func (s SyncSpec) GetImagePullPolicy() v1.PullPolicy { - return util.PullPolicyOrDefault(s.ImagePullPolicy) -} - // Validate the given spec func (s SyncSpec) Validate(mode DeploymentMode) error { if s.IsEnabled() && !mode.SupportsSync() { return maskAny(errors.Wrapf(ValidationError, "Cannot enable sync with mode: '%s'", mode)) } - if s.GetImage() == "" { - return maskAny(errors.Wrapf(ValidationError, "image must be set")) - } if s.IsEnabled() { if err := s.ExternalAccess.Validate(); err != nil { return maskAny(err) @@ -82,17 +66,11 @@ func (s SyncSpec) Validate(mode DeploymentMode) error { } // SetDefaults fills in missing defaults -func (s *SyncSpec) SetDefaults(defaultImage string, defaulPullPolicy v1.PullPolicy, defaultJWTSecretName, defaultClientAuthCASecretName, defaultTLSCASecretName string) { - if s.GetImage() == "" { - s.Image = util.NewString(defaultImage) - } - if s.GetImagePullPolicy() == "" { - s.ImagePullPolicy = util.NewPullPolicy(defaulPullPolicy) - } +func (s *SyncSpec) SetDefaults(defaultJWTSecretName, defaultClientAuthCASecretName, defaultTLSCASecretName, defaultMonitoringSecretName string) { s.ExternalAccess.SetDefaults() s.Authentication.SetDefaults(defaultJWTSecretName, defaultClientAuthCASecretName) s.TLS.SetDefaults(defaultTLSCASecretName) - s.Monitoring.SetDefaults() + s.Monitoring.SetDefaults(defaultMonitoringSecretName) } // SetDefaultsFrom fills unspecified fields with a value from given source spec. @@ -100,12 +78,6 @@ func (s *SyncSpec) SetDefaultsFrom(source SyncSpec) { if s.Enabled == nil { s.Enabled = util.NewBoolOrNil(source.Enabled) } - if s.Image == nil { - s.Image = util.NewStringOrNil(source.Image) - } - if s.ImagePullPolicy == nil { - s.ImagePullPolicy = util.NewPullPolicyOrNil(source.ImagePullPolicy) - } s.ExternalAccess.SetDefaultsFrom(source.ExternalAccess) s.Authentication.SetDefaultsFrom(source.Authentication) s.TLS.SetDefaultsFrom(source.TLS) diff --git a/pkg/apis/deployment/v1alpha/sync_spec_test.go b/pkg/apis/deployment/v1alpha/sync_spec_test.go index eb0d76f07..6d0b18f08 100644 --- a/pkg/apis/deployment/v1alpha/sync_spec_test.go +++ b/pkg/apis/deployment/v1alpha/sync_spec_test.go @@ -27,40 +27,33 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" ) func TestSyncSpecValidate(t *testing.T) { // Valid auth := SyncAuthenticationSpec{JWTSecretName: util.NewString("foo"), ClientCASecretName: util.NewString("foo-client")} tls := TLSSpec{CASecretName: util.NewString("None")} - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth}.Validate(DeploymentModeSingle)) - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth}.Validate(DeploymentModeActiveFailover)) - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth}.Validate(DeploymentModeCluster)) - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeCluster)) + assert.Nil(t, SyncSpec{Authentication: auth}.Validate(DeploymentModeSingle)) + assert.Nil(t, SyncSpec{Authentication: auth}.Validate(DeploymentModeActiveFailover)) + assert.Nil(t, SyncSpec{Authentication: auth}.Validate(DeploymentModeCluster)) + assert.Nil(t, SyncSpec{Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeCluster)) // Not valid - assert.Error(t, SyncSpec{Image: util.NewString(""), Authentication: auth}.Validate(DeploymentModeSingle)) - assert.Error(t, SyncSpec{Image: util.NewString(""), Authentication: auth}.Validate(DeploymentModeActiveFailover)) - assert.Error(t, SyncSpec{Image: util.NewString(""), Authentication: auth}.Validate(DeploymentModeCluster)) - assert.Error(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeSingle)) - assert.Error(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeActiveFailover)) + assert.Error(t, SyncSpec{Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeSingle)) + assert.Error(t, SyncSpec{Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeActiveFailover)) } func TestSyncSpecSetDefaults(t *testing.T) { def := func(spec SyncSpec) SyncSpec { - spec.SetDefaults("test-image", v1.PullAlways, "test-jwt", "test-client-auth-ca", "test-tls-ca") + spec.SetDefaults("test-jwt", "test-client-auth-ca", "test-tls-ca", "test-mon") return spec } assert.False(t, def(SyncSpec{}).IsEnabled()) assert.False(t, def(SyncSpec{Enabled: util.NewBool(false)}).IsEnabled()) assert.True(t, def(SyncSpec{Enabled: util.NewBool(true)}).IsEnabled()) - assert.Equal(t, "test-image", def(SyncSpec{}).GetImage()) - assert.Equal(t, "foo", def(SyncSpec{Image: util.NewString("foo")}).GetImage()) - assert.Equal(t, v1.PullAlways, def(SyncSpec{}).GetImagePullPolicy()) - assert.Equal(t, v1.PullNever, def(SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullNever)}).GetImagePullPolicy()) assert.Equal(t, "test-jwt", def(SyncSpec{}).Authentication.GetJWTSecretName()) + assert.Equal(t, "test-mon", def(SyncSpec{}).Monitoring.GetTokenSecretName()) assert.Equal(t, "foo", def(SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo")}}).Authentication.GetJWTSecretName()) } @@ -84,18 +77,6 @@ func TestSyncSpecResetImmutableFields(t *testing.T) { SyncSpec{Enabled: util.NewBool(false)}, nil, }, - { - SyncSpec{Image: util.NewString("foo")}, - SyncSpec{Image: util.NewString("foo2")}, - SyncSpec{Image: util.NewString("foo2")}, - nil, - }, - { - SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullAlways)}, - SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullNever)}, - SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullNever)}, - nil, - }, { SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("None"), ClientCASecretName: util.NewString("some")}}, SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("None"), ClientCASecretName: util.NewString("some")}}, diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 91e45a3c5..e0f332710 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -678,24 +678,6 @@ func (in *SyncSpec) DeepCopyInto(out *SyncSpec) { **out = **in } } - if in.Image != nil { - in, out := &in.Image, &out.Image - if *in == nil { - *out = nil - } else { - *out = new(string) - **out = **in - } - } - if in.ImagePullPolicy != nil { - in, out := &in.ImagePullPolicy, &out.ImagePullPolicy - if *in == nil { - *out = nil - } else { - *out = new(core_v1.PullPolicy) - **out = **in - } - } in.ExternalAccess.DeepCopyInto(&out.ExternalAccess) in.Authentication.DeepCopyInto(&out.Authentication) in.TLS.DeepCopyInto(&out.TLS) diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index f3e56ad16..5f0f609c3 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -25,6 +25,8 @@ package deployment import ( "context" + "github.com/arangodb/arangosync/client" + "github.com/arangodb/arangosync/tasks" driver "github.com/arangodb/go-driver" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -119,6 +121,38 @@ func (d *Deployment) GetAgencyClients(ctx context.Context, predicate func(id str return result, nil } +// GetSyncServerClient returns a cached client for a specific arangosync server. +func (d *Deployment) GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) { + // Fetch monitoring token + log := d.deps.Log + kubecli := d.deps.KubeCli + ns := d.apiObject.GetNamespace() + secretName := d.apiObject.Spec.Sync.Monitoring.GetTokenSecretName() + monitoringToken, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) + if err != nil { + log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") + return nil, maskAny(err) + } + + // Fetch server DNS name + dnsName := k8sutil.CreatePodDNSName(d.apiObject, group.AsRole(), id) + + // Build client + source := client.Endpoint{dnsName} + tlsAuth := tasks.TLSAuthentication{ + TLSClientAuthentication: tasks.TLSClientAuthentication{ + ClientToken: monitoringToken, + }, + } + auth := client.NewAuthentication(tlsAuth, "") + insecureSkipVerify := true + c, err := d.syncClientCache.GetClient(d.deps.Log, source, auth, insecureSkipVerify) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. func (d *Deployment) CreateMember(group api.ServerGroup, id string) error { diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 4dbe57a1e..aa17ad492 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -28,6 +28,7 @@ import ( "sync/atomic" "time" + "github.com/arangodb/arangosync/client" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "k8s.io/api/core/v1" @@ -101,6 +102,7 @@ type Deployment struct { resilience *resilience.Resilience resources *resources.Resources chaosMonkey *chaos.Monkey + syncClientCache client.ClientCache } // New creates a new Deployment from the given API object. @@ -280,6 +282,7 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { } newAPIObject := current.DeepCopy() newAPIObject.Spec.SetDefaultsFrom(specBefore) + newAPIObject.Spec.SetDefaults(d.apiObject.GetName()) newAPIObject.Status = d.status resetFields := specBefore.ResetImmutableFields(&newAPIObject.Spec) if len(resetFields) > 0 { diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index 86ff3ceb4..218dc1003 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -26,6 +26,7 @@ import ( "context" "fmt" + "github.com/arangodb/arangosync/client" driver "github.com/arangodb/go-driver" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -45,6 +46,8 @@ type ActionContext interface { GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) // GetAgencyClients returns a client connection for every agency member. GetAgencyClients(ctx context.Context) ([]driver.Connection, error) + // GetSyncServerClient returns a cached client for a specific arangosync server. + GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) // GetMemberStatusByID returns the current member status // for the member with given id. // Returns member status, true when found, or false @@ -115,6 +118,15 @@ func (ac *actionContext) GetAgencyClients(ctx context.Context) ([]driver.Connect return c, nil } +// GetSyncServerClient returns a cached client for a specific arangosync server. +func (ac *actionContext) GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) { + c, err := ac.context.GetSyncServerClient(ctx, group, id) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + // GetMemberStatusByID returns the current member status // for the member with given id. // Returns member status, true when found, or false diff --git a/pkg/deployment/reconcile/action_wait_for_member_up.go b/pkg/deployment/reconcile/action_wait_for_member_up.go index 2cf2dd922..e95bc286b 100644 --- a/pkg/deployment/reconcile/action_wait_for_member_up.go +++ b/pkg/deployment/reconcile/action_wait_for_member_up.go @@ -152,6 +152,15 @@ func (a *actionWaitForMemberUp) checkProgressCluster(ctx context.Context) (bool, // checkProgressArangoSync checks the progress of the action in the case // of a sync master / worker. func (a *actionWaitForMemberUp) checkProgressArangoSync(ctx context.Context) (bool, error) { - // TODO + log := a.log + c, err := a.actionCtx.GetSyncServerClient(ctx, a.action.Group, a.action.ID) + if err != nil { + log.Debug().Err(err).Msg("Failed to create arangosync client") + return false, maskAny(err) + } + if err := c.Health(ctx); err != nil { + log.Debug().Err(err).Msg("Health not ok yet") + return false, maskAny(err) + } return true, nil } diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index cfa460b94..342de904b 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -25,6 +25,7 @@ package reconcile import ( "context" + "github.com/arangodb/arangosync/client" driver "github.com/arangodb/go-driver" "k8s.io/api/core/v1" @@ -51,6 +52,8 @@ type Context interface { // GetAgencyClients returns a client connection for every agency member. // If the given predicate is not nil, only agents are included where the given predicate returns true. GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]driver.Connection, error) + // GetSyncServerClient returns a cached client for a specific arangosync server. + GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. CreateMember(group api.ServerGroup, id string) error diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 7bc8b3a6f..6026a639c 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -128,9 +128,12 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, // Only scale singles plan = append(plan, createScalePlan(log, status.Members.Single, api.ServerGroupSingle, spec.Single.GetCount())...) case api.DeploymentModeCluster: - // Scale dbservers, coordinators, syncmasters & syncworkers + // Scale dbservers, coordinators plan = append(plan, createScalePlan(log, status.Members.DBServers, api.ServerGroupDBServers, spec.DBServers.GetCount())...) plan = append(plan, createScalePlan(log, status.Members.Coordinators, api.ServerGroupCoordinators, spec.Coordinators.GetCount())...) + } + if spec.GetMode().SupportsSync() { + // Scale syncmasters & syncworkers plan = append(plan, createScalePlan(log, status.Members.SyncMasters, api.ServerGroupSyncMasters, spec.SyncMasters.GetCount())...) plan = append(plan, createScalePlan(log, status.Members.SyncWorkers, api.ServerGroupSyncWorkers, spec.SyncWorkers.GetCount())...) } diff --git a/pkg/deployment/resources/certificates_client_auth.go b/pkg/deployment/resources/certificates_client_auth.go new file mode 100644 index 000000000..b5e494632 --- /dev/null +++ b/pkg/deployment/resources/certificates_client_auth.go @@ -0,0 +1,113 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "fmt" + "strings" + "time" + + certificates "github.com/arangodb-helper/go-certificates" + "github.com/rs/zerolog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/typed/core/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +const ( + clientAuthECDSACurve = "P256" // This curve is the default that ArangoDB accepts and plenty strong +) + +// createClientAuthCACertificate creates a client authentication CA certificate and stores it in a secret with name +// specified in the given spec. +func createClientAuthCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.SyncAuthenticationSpec, deploymentName, namespace string, ownerRef *metav1.OwnerReference) error { + log = log.With().Str("secret", spec.GetClientCASecretName()).Logger() + options := certificates.CreateCertificateOptions{ + CommonName: fmt.Sprintf("%s Client Authentication Root Certificate", deploymentName), + ValidFrom: time.Now(), + ValidFor: caTTL, + IsCA: true, + IsClientAuth: true, + ECDSACurve: clientAuthECDSACurve, + } + cert, priv, err := certificates.CreateCertificate(options, nil) + if err != nil { + log.Debug().Err(err).Msg("Failed to create CA certificate") + return maskAny(err) + } + if err := k8sutil.CreateCASecret(cli, spec.GetClientCASecretName(), namespace, cert, priv, ownerRef); err != nil { + if k8sutil.IsAlreadyExists(err) { + log.Debug().Msg("CA Secret already exists") + } else { + log.Debug().Err(err).Msg("Failed to create CA Secret") + } + return maskAny(err) + } + log.Debug().Msg("Created CA Secret") + return nil +} + +// createClientAuthCertificateKeyfile creates a client authentication certificate for a specific user and stores +// it in a secret with the given name. +func createClientAuthCertificateKeyfile(log zerolog.Logger, cli v1.CoreV1Interface, commonName string, ttl time.Duration, spec api.SyncAuthenticationSpec, secretName, namespace string, ownerRef *metav1.OwnerReference) error { + log = log.With().Str("secret", secretName).Logger() + // Load CA certificate + caCert, caKey, err := k8sutil.GetCASecret(cli, spec.GetClientCASecretName(), namespace) + if err != nil { + log.Debug().Err(err).Msg("Failed to load CA certificate") + return maskAny(err) + } + ca, err := certificates.LoadCAFromPEM(caCert, caKey) + if err != nil { + log.Debug().Err(err).Msg("Failed to decode CA certificate") + return maskAny(err) + } + + options := certificates.CreateCertificateOptions{ + CommonName: commonName, + ValidFrom: time.Now(), + ValidFor: ttl, + IsCA: false, + IsClientAuth: true, + ECDSACurve: clientAuthECDSACurve, + } + cert, priv, err := certificates.CreateCertificate(options, &ca) + if err != nil { + log.Debug().Err(err).Msg("Failed to create server certificate") + return maskAny(err) + } + keyfile := strings.TrimSpace(cert) + "\n" + + strings.TrimSpace(priv) + if err := k8sutil.CreateTLSKeyfileSecret(cli, secretName, namespace, keyfile, ownerRef); err != nil { + if k8sutil.IsAlreadyExists(err) { + log.Debug().Msg("Server Secret already exists") + } else { + log.Debug().Err(err).Msg("Failed to create server Secret") + } + return maskAny(err) + } + log.Debug().Msg("Created server Secret") + return nil +} diff --git a/pkg/deployment/resources/tls.go b/pkg/deployment/resources/certificates_tls.go similarity index 88% rename from pkg/deployment/resources/tls.go rename to pkg/deployment/resources/certificates_tls.go index 725fa0c40..f0e5e8d88 100644 --- a/pkg/deployment/resources/tls.go +++ b/pkg/deployment/resources/certificates_tls.go @@ -41,9 +41,9 @@ const ( tlsECDSACurve = "P256" // This curve is the default that ArangoDB accepts and plenty strong ) -// createCACertificate creates a CA certificate and stores it in a secret with name +// createTLSCACertificate creates a CA certificate and stores it in a secret with name // specified in the given spec. -func createCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.TLSSpec, deploymentName, namespace string, ownerRef *metav1.OwnerReference) error { +func createTLSCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.TLSSpec, deploymentName, namespace string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", spec.GetCASecretName()).Logger() dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames() if err != nil { @@ -77,9 +77,9 @@ func createCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.TL return nil } -// createServerCertificate creates a TLS certificate for a specific server and stores +// createTLSServerCertificate creates a TLS certificate for a specific server and stores // it in a secret with the given name. -func createServerCertificate(log zerolog.Logger, cli v1.CoreV1Interface, serverNames []string, spec api.TLSSpec, secretName, namespace string, ownerRef *metav1.OwnerReference) error { +func createTLSServerCertificate(log zerolog.Logger, cli v1.CoreV1Interface, serverNames []string, spec api.TLSSpec, secretName, namespace string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", secretName).Logger() // Load alt names dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames() diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 0505cdff9..0241048fa 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -234,7 +234,7 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou optionPair{"--monitoring.token", "$(" + constants.EnvArangoSyncMonitoringToken + ")"}, ) } - masterSecretPath := filepath.Join(k8sutil.MasterJWTSecretVolumeMountDir, constants.SecretKeyJWT) + masterSecretPath := filepath.Join(k8sutil.MasterJWTSecretVolumeMountDir, constants.SecretKeyToken) options = append(options, optionPair{"--master.jwt-secret", masterSecretPath}, ) @@ -252,7 +252,7 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou optionPair{"--mq.type", "direct"}, ) if spec.IsAuthenticated() { - clusterSecretPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyJWT) + clusterSecretPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken) options = append(options, optionPair{"--cluster.jwt-secret", clusterSecretPath}, ) @@ -327,7 +327,7 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve if err != nil { return nil, maskAny(err) } - authorization = "bearer: " + token + authorization = "bearer " + token if err != nil { return nil, maskAny(err) } @@ -445,14 +445,14 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server podSuffix := createPodSuffix(spec) m.PodName = k8sutil.CreatePodName(apiObject.GetName(), roleAbbr, m.ID, podSuffix) newPhase := api.MemberPhaseCreated + // Find image ID + imageInfo, imageFound := status.Images.GetByImage(spec.GetImage()) + if !imageFound { + log.Debug().Str("image", spec.GetImage()).Msg("Image ID is not known yet for image") + return nil + } // Create pod if group.IsArangod() { - // Find image ID - info, found := status.Images.GetByImage(spec.GetImage()) - if !found { - log.Debug().Str("image", spec.GetImage()).Msg("Image ID is not known yet for image") - return nil - } // Prepare arguments autoUpgrade := m.Conditions.IsTrue(api.ConditionTypeAutoUpgrade) if autoUpgrade { @@ -479,7 +479,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server serverNames = append(serverNames, ip) } owner := apiObject.AsOwner() - if err := createServerCertificate(log, kubecli.CoreV1(), serverNames, spec.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + if err := createTLSServerCertificate(log, kubecli.CoreV1(), serverNames, spec.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret")) } } @@ -493,34 +493,34 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server if spec.IsAuthenticated() { env[constants.EnvArangodJWTSecret] = k8sutil.EnvValue{ SecretName: spec.Authentication.GetJWTSecretName(), - SecretKey: constants.SecretKeyJWT, + SecretKey: constants.SecretKeyToken, } } engine := spec.GetStorageEngine().AsArangoArgument() requireUUID := group == api.ServerGroupDBServers && m.IsInitialized finalizers := r.createPodFinalizers(group) - if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, lifecycleImage, spec.GetImagePullPolicy(), + if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, imageInfo.ImageID, lifecycleImage, spec.GetImagePullPolicy(), engine, requireUUID, terminationGracePeriod, args, env, finalizers, livenessProbe, readinessProbe, tolerations, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") } else if group.IsArangosync() { - // Find image ID - info, found := status.Images.GetByImage(spec.Sync.GetImage()) - if !found { - log.Debug().Str("image", spec.Sync.GetImage()).Msg("Image ID is not known yet for sync image") - return nil - } - if !info.Enterprise { - log.Debug().Str("image", spec.Sync.GetImage()).Msg("Image is not an enterprise image") - return maskAny(fmt.Errorf("Image '%s' does not contain an Enterprise version of ArangoDB", spec.Sync.GetImage())) + // Check image + if !imageInfo.Enterprise { + log.Debug().Str("image", spec.GetImage()).Msg("Image is not an enterprise image") + return maskAny(fmt.Errorf("Image '%s' does not contain an Enterprise version of ArangoDB", spec.GetImage())) } var tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName string // Check master JWT secret masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() - if err := k8sutil.ValidateJWTSecret(kubecli.CoreV1(), masterJWTSecretName, ns); err != nil { + if err := k8sutil.ValidateTokenSecret(kubecli.CoreV1(), masterJWTSecretName, ns); err != nil { return maskAny(errors.Wrapf(err, "Master JWT secret validation failed")) } + // Check monitoring token secret + monitoringTokenSecretName := spec.Sync.Monitoring.GetTokenSecretName() + if err := k8sutil.ValidateTokenSecret(kubecli.CoreV1(), monitoringTokenSecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Monitoring token secret validation failed")) + } if group == api.ServerGroupSyncMasters { // Create TLS secret tlsKeyfileSecretName = k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID) @@ -536,13 +536,13 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server } } owner := apiObject.AsOwner() - if err := createServerCertificate(log, kubecli.CoreV1(), serverNames, spec.Sync.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + if err := createTLSServerCertificate(log, kubecli.CoreV1(), serverNames, spec.Sync.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret")) } // Check cluster JWT secret if spec.IsAuthenticated() { clusterJWTSecretName = spec.Authentication.GetJWTSecretName() - if err := k8sutil.ValidateJWTSecret(kubecli.CoreV1(), clusterJWTSecretName, ns); err != nil { + if err := k8sutil.ValidateTokenSecret(kubecli.CoreV1(), clusterJWTSecretName, ns); err != nil { return maskAny(errors.Wrapf(err, "Cluster JWT secret validation failed")) } } @@ -559,7 +559,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server if spec.Sync.Monitoring.GetTokenSecretName() != "" { env[constants.EnvArangoSyncMonitoringToken] = k8sutil.EnvValue{ SecretName: spec.Sync.Monitoring.GetTokenSecretName(), - SecretKey: constants.SecretKeyJWT, + SecretKey: constants.SecretKeyToken, } } livenessProbe, err := r.createLivenessProbe(spec, group) @@ -570,7 +570,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server if group == api.ServerGroupSyncWorkers { affinityWithRole = api.ServerGroupDBServers.AsRole() } - if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, info.ImageID, lifecycleImage, spec.Sync.GetImagePullPolicy(), terminationGracePeriod, args, env, + if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, imageInfo.ImageID, lifecycleImage, spec.GetImagePullPolicy(), terminationGracePeriod, args, env, livenessProbe, tolerations, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole); err != nil { return maskAny(err) } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index b59e0aeb3..205736a43 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -25,7 +25,6 @@ package resources import ( "crypto/rand" "encoding/hex" - "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,30 +36,36 @@ import ( func (r *Resources) EnsureSecrets() error { spec := r.context.GetSpec() if spec.IsAuthenticated() { - if err := r.ensureJWTSecret(spec.Authentication.GetJWTSecretName()); err != nil { + if err := r.ensureTokenSecret(spec.Authentication.GetJWTSecretName()); err != nil { return maskAny(err) } } if spec.IsSecure() { - if err := r.ensureCACertificateSecret(spec.TLS); err != nil { + if err := r.ensureTLSCACertificateSecret(spec.TLS); err != nil { return maskAny(err) } } if spec.Sync.IsEnabled() { - if err := r.ensureJWTSecret(spec.Sync.Authentication.GetJWTSecretName()); err != nil { + if err := r.ensureTokenSecret(spec.Sync.Authentication.GetJWTSecretName()); err != nil { return maskAny(err) } - if err := r.ensureCACertificateSecret(spec.Sync.TLS); err != nil { + if err := r.ensureTokenSecret(spec.Sync.Monitoring.GetTokenSecretName()); err != nil { + return maskAny(err) + } + if err := r.ensureTLSCACertificateSecret(spec.Sync.TLS); err != nil { + return maskAny(err) + } + if err := r.ensureClientAuthCACertificateSecret(spec.Sync.Authentication); err != nil { return maskAny(err) } } return nil } -// ensureJWTSecret checks if a secret with given name exists in the namespace +// ensureTokenSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a random -// JWT token. -func (r *Resources) ensureJWTSecret(secretName string) error { +// token. +func (r *Resources) ensureTokenSecret(secretName string) error { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() if _, err := kubecli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) { @@ -72,7 +77,7 @@ func (r *Resources) ensureJWTSecret(secretName string) error { // Create secret owner := r.context.GetAPIObject().AsOwner() - if err := k8sutil.CreateJWTSecret(kubecli.CoreV1(), secretName, ns, token, &owner); k8sutil.IsAlreadyExists(err) { + if err := k8sutil.CreateTokenSecret(kubecli.CoreV1(), secretName, ns, token, &owner); k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -86,10 +91,9 @@ func (r *Resources) ensureJWTSecret(secretName string) error { return nil } -// ensureCACertificateSecret checks if a secret with given name exists in the namespace +// ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -// JWT token. -func (r *Resources) ensureCACertificateSecret(spec api.TLSSpec) error { +func (r *Resources) ensureTLSCACertificateSecret(spec api.TLSSpec) error { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() if _, err := kubecli.CoreV1().Secrets(ns).Get(spec.GetCASecretName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) { @@ -97,7 +101,31 @@ func (r *Resources) ensureCACertificateSecret(spec api.TLSSpec) error { apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - if err := createCACertificate(r.log, kubecli.CoreV1(), spec, deploymentName, ns, &owner); k8sutil.IsAlreadyExists(err) { + if err := createTLSCACertificate(r.log, kubecli.CoreV1(), spec, deploymentName, ns, &owner); k8sutil.IsAlreadyExists(err) { + // Secret added while we tried it also + return nil + } else if err != nil { + // Failed to create secret + return maskAny(err) + } + } else if err != nil { + // Failed to get secret for other reasons + return maskAny(err) + } + return nil +} + +// ensureClientAuthCACertificateSecret checks if a secret with given name exists in the namespace +// of the deployment. If not, it will add such a secret with a generated CA certificate. +func (r *Resources) ensureClientAuthCACertificateSecret(spec api.SyncAuthenticationSpec) error { + kubecli := r.context.GetKubeCli() + ns := r.context.GetNamespace() + if _, err := kubecli.CoreV1().Secrets(ns).Get(spec.GetClientCASecretName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) { + // Secret not found, create it + apiObject := r.context.GetAPIObject() + owner := apiObject.AsOwner() + deploymentName := apiObject.GetName() + if err := createClientAuthCACertificate(r.log, kubecli.CoreV1(), spec, deploymentName, ns, &owner); k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -119,7 +147,7 @@ func (r *Resources) getJWTSecret(spec api.DeploymentSpec) (string, error) { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() secretName := spec.Authentication.GetJWTSecretName() - s, err := k8sutil.GetJWTSecret(kubecli.CoreV1(), secretName, ns) + s, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get JWT secret") return "", maskAny(err) @@ -132,7 +160,7 @@ func (r *Resources) getSyncJWTSecret(spec api.DeploymentSpec) (string, error) { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() secretName := spec.Sync.Authentication.GetJWTSecretName() - s, err := k8sutil.GetJWTSecret(kubecli.CoreV1(), secretName, ns) + s, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync JWT secret") return "", maskAny(err) @@ -145,13 +173,10 @@ func (r *Resources) getSyncMonitoringToken(spec api.DeploymentSpec) (string, err kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() secretName := spec.Sync.Monitoring.GetTokenSecretName() - s, err := kubecli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}) + s, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) if err != nil { - r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get monitoring token secret") - } - // Take the first data - for _, v := range s.Data { - return string(v), nil + r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") + return "", maskAny(err) } - return "", maskAny(fmt.Errorf("No data found in secret '%s'", secretName)) + return s, nil } diff --git a/pkg/replication/sync_client.go b/pkg/replication/sync_client.go index a1edf3121..ef31ce64b 100644 --- a/pkg/replication/sync_client.go +++ b/pkg/replication/sync_client.go @@ -55,7 +55,7 @@ func (dr *DeploymentReplication) createSyncMasterClient(epSpec api.EndpointSpec) jwtSecret := "" if authJWTSecretName != "" { var err error - jwtSecret, err = k8sutil.GetJWTSecret(dr.deps.KubeCli.CoreV1(), authJWTSecretName, dr.apiObject.GetNamespace()) + jwtSecret, err = k8sutil.GetTokenSecret(dr.deps.KubeCli.CoreV1(), authJWTSecretName, dr.apiObject.GetNamespace()) if err != nil { return nil, maskAny(err) } diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 655fe27b0..25526f69e 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -148,7 +148,7 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa // Authentication is enabled. // Should we skip using it? if ctx.Value(skipAuthenticationKey{}) == nil { - s, err := k8sutil.GetJWTSecret(cli, apiObject.Spec.Authentication.GetJWTSecretName(), apiObject.GetNamespace()) + s, err := k8sutil.GetTokenSecret(cli, apiObject.Spec.Authentication.GetJWTSecretName(), apiObject.GetNamespace()) if err != nil { return nil, maskAny(err) } diff --git a/pkg/util/constants/constants.go b/pkg/util/constants/constants.go index dcf0d471c..8307516b7 100644 --- a/pkg/util/constants/constants.go +++ b/pkg/util/constants/constants.go @@ -32,8 +32,7 @@ const ( EnvArangoSyncMonitoringToken = "ARANGOSYNC_MONITORING_TOKEN" // Constains monitoring token for ArangoSync servers SecretEncryptionKey = "key" // Key in a Secret.Data used to store an 32-byte encryption key - SecretKeyJWT = "token" // Key inside a Secret used to hold a JW token - SecretKeyMonitoring = "token" // Key inside a Secret used to hold a monitoring token + SecretKeyToken = "token" // Key inside a Secret used to hold a JWT or monitoring token SecretCACertificate = "ca.crt" // Key in Secret.data used to store a PEM encoded CA certificate (public key) SecretCAKey = "ca.key" // Key in Secret.data used to store a PEM encoded CA private key diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index b1b358a7a..6acf0e81c 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -184,45 +184,45 @@ func CreateTLSKeyfileSecret(cli corev1.CoreV1Interface, secretName, namespace st return nil } -// ValidateJWTSecret checks that a secret with given name in given namespace +// ValidateTokenSecret checks that a secret with given name in given namespace // exists and it contains a 'token' data field. -func ValidateJWTSecret(cli corev1.CoreV1Interface, secretName, namespace string) error { +func ValidateTokenSecret(cli corev1.CoreV1Interface, secretName, namespace string) error { s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) if err != nil { return maskAny(err) } // Check `token` field - _, found := s.Data[constants.SecretKeyJWT] + _, found := s.Data[constants.SecretKeyToken] if !found { - return maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretKeyJWT, secretName)) + return maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretKeyToken, secretName)) } return nil } -// GetJWTSecret loads the JWT secret from a Secret with given name. -func GetJWTSecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, error) { +// GetTokenSecret loads the token secret from a Secret with given name. +func GetTokenSecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, error) { s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) if err != nil { return "", maskAny(err) } // Take the first data from the token key - data, found := s.Data[constants.SecretKeyJWT] + data, found := s.Data[constants.SecretKeyToken] if !found { - return "", maskAny(fmt.Errorf("No '%s' data found in secret '%s'", constants.SecretKeyJWT, secretName)) + return "", maskAny(fmt.Errorf("No '%s' data found in secret '%s'", constants.SecretKeyToken, secretName)) } return string(data), nil } -// CreateJWTSecret creates a secret with given name in given namespace +// CreateTokenSecret creates a secret with given name in given namespace // with a given token as value. -func CreateJWTSecret(cli corev1.CoreV1Interface, secretName, namespace, token string, ownerRef *metav1.OwnerReference) error { +func CreateTokenSecret(cli corev1.CoreV1Interface, secretName, namespace, token string, ownerRef *metav1.OwnerReference) error { // Create secret secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, }, Data: map[string][]byte{ - constants.SecretKeyJWT: []byte(token), + constants.SecretKeyToken: []byte(token), }, } // Attach secret to owner diff --git a/pkg/util/k8sutil/secrets_test.go b/pkg/util/k8sutil/secrets_test.go index 18cf85216..476974ab3 100644 --- a/pkg/util/k8sutil/secrets_test.go +++ b/pkg/util/k8sutil/secrets_test.go @@ -86,8 +86,8 @@ func TestCreateEncryptionKeySecret(t *testing.T) { assert.Error(t, CreateEncryptionKeySecret(cli, "short-key", "ns", key)) } -// TestGetJWTSecret tests GetJWTSecret. -func TestGetJWTSecret(t *testing.T) { +// TestGetTokenSecret tests GetTokenSecret. +func TestGetTokenSecret(t *testing.T) { cli := mocks.NewCore() // Prepare mock @@ -104,17 +104,17 @@ func TestGetJWTSecret(t *testing.T) { }, nil) m.On("Get", "notfound", mock.Anything).Return(nil, apierrors.NewNotFound(schema.GroupResource{}, "notfound")) - token, err := GetJWTSecret(cli, "good", "ns") + token, err := GetTokenSecret(cli, "good", "ns") assert.NoError(t, err) assert.Equal(t, token, "foo") - _, err = GetJWTSecret(cli, "no-token", "ns") + _, err = GetTokenSecret(cli, "no-token", "ns") assert.Error(t, err) - _, err = GetJWTSecret(cli, "notfound", "ns") + _, err = GetTokenSecret(cli, "notfound", "ns") assert.True(t, IsNotFound(err)) } -// TestCreateJWTSecret tests CreateJWTSecret -func TestCreateJWTSecret(t *testing.T) { +// TestCreateTokenSecret tests CreateTokenSecret +func TestCreateTokenSecret(t *testing.T) { cli := mocks.NewCore() // Prepare mock @@ -130,6 +130,6 @@ func TestCreateJWTSecret(t *testing.T) { } }).Return(nil, nil) - assert.NoError(t, CreateJWTSecret(cli, "good", "ns", "token", nil)) - assert.NoError(t, CreateJWTSecret(cli, "with-owner", "ns", "token", &metav1.OwnerReference{})) + assert.NoError(t, CreateTokenSecret(cli, "good", "ns", "token", nil)) + assert.NoError(t, CreateTokenSecret(cli, "with-owner", "ns", "token", &metav1.OwnerReference{})) } diff --git a/tests/auth_test.go b/tests/auth_test.go index 66c2b9fb7..611190989 100644 --- a/tests/auth_test.go +++ b/tests/auth_test.go @@ -100,7 +100,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateJWTSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } @@ -239,7 +239,7 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateJWTSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } diff --git a/tests/scale_test.go b/tests/scale_test.go index 6e8bc3079..38b1a0998 100644 --- a/tests/scale_test.go +++ b/tests/scale_test.go @@ -177,3 +177,105 @@ func TestScaleCluster(t *testing.T) { // Cleanup removeDeployment(c, depl.GetName(), ns) } + +// TestScaleClusterWithSync tests scaling a cluster deployment with sync enabled. +func TestScaleClusterWithSync(t *testing.T) { + longOrSkip(t) + img := getEnterpriseImageOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-scale-sync" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Image = util.NewString(img) + depl.Spec.Sync.Enabled = util.NewBool(true) + depl.Spec.SetDefaults(depl.GetName()) // this must be last + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer deferedCleanupDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Create a syncmaster client + syncClient := mustNewArangoSyncClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be completely ready + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, apiObject.Spec) + }); err != nil { + t.Fatalf("Cluster not running in expected health in time: %v", err) + } + + // Wait for syncmasters to be available + if err := waitUntilSyncVersionUp(syncClient, nil); err != nil { + t.Fatalf("SyncMasters not running returning version in time: %v", err) + } + + // Add 1 DBServer, 2 SyncMasters, 1 syncworker + updated, err := updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.DBServers.Count = util.NewInt(4) + spec.SyncMasters.Count = util.NewInt(5) + spec.SyncWorkers.Count = util.NewInt(4) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait for cluster to reach new size + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, updated.Spec) + }); err != nil { + t.Fatalf("Cluster not running, after scale-up, in expected health in time: %v", err) + } + // Check number of syncmasters + if err := waitUntilSyncMasterCountReached(syncClient, updated.Spec.SyncMasters.GetCount()); err != nil { + t.Fatalf("Unexpected #syncmasters, after scale-up: %v", err) + } + // Check number of syncworkers + if err := waitUntilSyncWorkerCountReached(syncClient, updated.Spec.SyncWorkers.GetCount()); err != nil { + t.Fatalf("Unexpected #syncworkers, after scale-up: %v", err) + } + + // Remove 1 DBServer, 2 SyncMasters & 1 SyncWorker + updated, err = updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.DBServers.Count = util.NewInt(3) + spec.SyncMasters.Count = util.NewInt(3) + spec.SyncWorkers.Count = util.NewInt(3) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait for cluster to reach new size + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, updated.Spec) + }); err != nil { + t.Fatalf("Cluster not running, after scale-down, in expected health in time: %v", err) + } + // Check number of syncmasters + if err := waitUntilSyncMasterCountReached(syncClient, updated.Spec.SyncMasters.GetCount()); err != nil { + t.Fatalf("Unexpected #syncmasters, after scale-up: %v", err) + } + // Check number of syncworkers + if err := waitUntilSyncWorkerCountReached(syncClient, updated.Spec.SyncWorkers.GetCount()); err != nil { + t.Fatalf("Unexpected #syncworkers, after scale-up: %v", err) + } + + // Cleanup + removeDeployment(c, depl.GetName(), ns) +} diff --git a/tests/simple_test.go b/tests/simple_test.go index 90427aa41..91e4dc5a6 100644 --- a/tests/simple_test.go +++ b/tests/simple_test.go @@ -32,6 +32,7 @@ import ( driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/client" + "github.com/arangodb/kube-arangodb/pkg/util" ) // TestSimpleSingle tests the creating of a single server deployment @@ -145,11 +146,63 @@ func TestSimpleCluster(t *testing.T) { ctx := context.Background() client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) - // Wait for single server available + // Wait for cluster to be available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Cluster not running returning version in time: %v", err) + } + + // Check server role + assert.NoError(t, client.SynchronizeEndpoints(ctx)) + role, err := client.ServerRole(ctx) + assert.NoError(t, err) + assert.Equal(t, driver.ServerRoleCoordinator, role) +} + +// TestSimpleClusterWithSync tests the creating of a cluster deployment +// with default settings and sync enabled. +func TestSimpleClusterWithSync(t *testing.T) { + img := getEnterpriseImageOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-cls-sync-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Image = util.NewString(img) + depl.Spec.Sync.Enabled = util.NewBool(true) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be available if err := waitUntilVersionUp(client, nil); err != nil { t.Fatalf("Cluster not running returning version in time: %v", err) } + // Create a syncmaster client + syncClient := mustNewArangoSyncClient(ctx, kubecli, apiObject, t) + + // Wait for syncmasters to be available + if err := waitUntilSyncVersionUp(syncClient, nil); err != nil { + t.Fatalf("SyncMasters not running returning version in time: %v", err) + } + // Check server role assert.NoError(t, client.SynchronizeEndpoints(ctx)) role, err := client.ServerRole(ctx) diff --git a/tests/sync_test.go b/tests/sync_test.go new file mode 100644 index 000000000..ec3ea4af4 --- /dev/null +++ b/tests/sync_test.go @@ -0,0 +1,136 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package tests + +import ( + "context" + "fmt" + "testing" + + "github.com/dchest/uniuri" + + driver "github.com/arangodb/go-driver" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/client" + "github.com/arangodb/kube-arangodb/pkg/util" +) + +// TestSyncToggleEnabled tests a normal cluster and enables sync later. +// Once sync is active, it is disabled again. +func TestSyncToggleEnabled(t *testing.T) { + longOrSkip(t) + img := getEnterpriseImageOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-sync-toggle-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Image = util.NewString(img) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer deferedCleanupDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be completely ready + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, apiObject.Spec) + }); err != nil { + t.Fatalf("Cluster not running in expected health in time: %v", err) + } + + // Enable sync + updated, err := updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.Sync.Enabled = util.NewBool(true) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait until sync jwt secret has been created + if _, err := waitUntilSecret(kubecli, updated.Spec.Sync.Authentication.GetJWTSecretName(), ns, nil, deploymentReadyTimeout); err != nil { + t.Fatalf("Sync JWT secret not created in time: %v", err) + } + + // Create a syncmaster client + syncClient := mustNewArangoSyncClient(ctx, kubecli, apiObject, t) + + // Wait for syncmasters to be available + if err := waitUntilSyncVersionUp(syncClient, nil); err != nil { + t.Fatalf("SyncMasters not running returning version in time: %v", err) + } + + // Wait for cluster to reach new size + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, updated.Spec) + }); err != nil { + t.Fatalf("Cluster not running, after scale-up, in expected health in time: %v", err) + } + // Check number of syncmasters + if err := waitUntilSyncMasterCountReached(syncClient, 3); err != nil { + t.Fatalf("Unexpected #syncmasters, after enabling sync: %v", err) + } + // Check number of syncworkers + if err := waitUntilSyncWorkerCountReached(syncClient, 3); err != nil { + t.Fatalf("Unexpected #syncworkers, after enabling sync: %v", err) + } + + // Disable sync + updated, err = updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.Sync.Enabled = util.NewBool(false) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait for deployment to have no more syncmasters & workers + if _, err := waitUntilDeployment(c, depl.GetName(), ns, func(apiObject *api.ArangoDeployment) error { + if cnt := len(apiObject.Status.Members.SyncMasters); cnt > 0 { + return maskAny(fmt.Errorf("Expected 0 syncmasters, got %d", cnt)) + } + if cnt := len(apiObject.Status.Members.SyncWorkers); cnt > 0 { + return maskAny(fmt.Errorf("Expected 0 syncworkers, got %d", cnt)) + } + return nil + }); err != nil { + t.Fatalf("Failed to reach deployment state without syncmasters & syncworkers: %v", err) + } + + // Cleanup + removeDeployment(c, depl.GetName(), ns) +} diff --git a/tests/test_util.go b/tests/test_util.go index 211cfed36..9463f8284 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -25,7 +25,9 @@ package tests import ( "context" "fmt" + "net" "os" + "strconv" "strings" "testing" "time" @@ -33,10 +35,13 @@ import ( "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" + "github.com/arangodb/arangosync/client" + "github.com/arangodb/arangosync/tasks" + driver "github.com/arangodb/go-driver" "github.com/pkg/errors" + "github.com/rs/zerolog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/util/arangod" @@ -49,7 +54,8 @@ const ( ) var ( - maskAny = errors.WithStack + maskAny = errors.WithStack + syncClientCache client.ClientCache ) // longOrSkip checks the short test flag. @@ -97,6 +103,32 @@ func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interf return c } +// mustNewArangoSyncClient creates a new arangosync client, with all syncmasters +// as endpoint. It is failing the test on errors. +func mustNewArangoSyncClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T) client.API { + ns := apiObject.GetNamespace() + secretName := apiObject.Spec.Sync.Authentication.GetJWTSecretName() + jwtToken, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) + if err != nil { + t.Fatalf("Failed to get sync jwt secret '%s': %s", secretName, err) + } + + // Fetch service DNS name + dnsName := k8sutil.CreateSyncMasterClientServiceDNSName(apiObject) + ep := client.Endpoint{"https://" + net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoSyncMasterPort))} + + // Build client + log := zerolog.Logger{} + tlsAuth := tasks.TLSAuthentication{} + auth := client.NewAuthentication(tlsAuth, jwtToken) + insecureSkipVerify := true + c, err := syncClientCache.GetClient(log, ep, auth, insecureSkipVerify) + if err != nil { + t.Fatalf("Failed to get sync client: %s", err) + } + return c +} + // getNamespace returns the kubernetes namespace in which to run tests. func getNamespace(t *testing.T) string { ns := os.Getenv("TEST_NAMESPACE") @@ -247,6 +279,70 @@ func waitUntilVersionUp(cli driver.Client, predicate func(driver.VersionInfo) er return nil } +// waitUntilSyncVersionUp waits until the syncmasters responds to +// an `/_api/version` request without an error. An additional Predicate +// can do a check on the VersionInfo object returned by the server. +func waitUntilSyncVersionUp(cli client.API, predicate func(client.VersionInfo) error) error { + ctx := context.Background() + + op := func() error { + if version, err := cli.Version(ctx); err != nil { + return maskAny(err) + } else if predicate != nil { + return predicate(version) + } + return nil + } + + if err := retry.Retry(op, deploymentReadyTimeout); err != nil { + return maskAny(err) + } + + return nil +} + +// waitUntilSyncMasterCountReached waits until the number of syncmasters +// is equal to the given number. +func waitUntilSyncMasterCountReached(cli client.API, expectedSyncMasters int) error { + ctx := context.Background() + + op := func() error { + if list, err := cli.Master().Masters(ctx); err != nil { + return maskAny(err) + } else if len(list) != expectedSyncMasters { + return maskAny(fmt.Errorf("Expected %d syncmasters, got %d", expectedSyncMasters, len(list))) + } + return nil + } + + if err := retry.Retry(op, deploymentReadyTimeout); err != nil { + return maskAny(err) + } + + return nil +} + +// waitUntilSyncWorkerCountReached waits until the number of syncworkers +// is equal to the given number. +func waitUntilSyncWorkerCountReached(cli client.API, expectedSyncWorkers int) error { + ctx := context.Background() + + op := func() error { + if list, err := cli.Master().RegisteredWorkers(ctx); err != nil { + return maskAny(err) + } else if len(list) != expectedSyncWorkers { + return maskAny(fmt.Errorf("Expected %d syncworkers, got %d", expectedSyncWorkers, len(list))) + } + return nil + } + + if err := retry.Retry(op, deploymentReadyTimeout); err != nil { + return maskAny(err) + } + + return nil +} + // creates predicate to be used in waitUntilVersionUp func createEqualVersionsPredicate(version driver.Version) func(driver.VersionInfo) error { return func(infoFromServer driver.VersionInfo) error {