From 0032058449ffa4d24f5cd2e9a30a77ddf8cfd764 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Fri, 24 Jul 2020 12:26:06 -0500 Subject: [PATCH 01/16] Initial NodePortConfig added --- .../templates/customresourcedefinition.yaml | 14 ++++ ...datastax.com_cassandradatacenters_crd.yaml | 14 ++++ .../v1beta1/cassandradatacenter_types.go | 83 +++++++++++++++++-- .../v1beta1/zz_generated.deepcopy.go | 42 ++++++++++ 4 files changed, 146 insertions(+), 7 deletions(-) diff --git a/charts/cass-operator-chart/templates/customresourcedefinition.yaml b/charts/cass-operator-chart/templates/customresourcedefinition.yaml index be3b5964c..512187a4b 100644 --- a/charts/cass-operator-chart/templates/customresourcedefinition.yaml +++ b/charts/cass-operator-chart/templates/customresourcedefinition.yaml @@ -92,6 +92,20 @@ spec: - serverSecretName type: object type: object + networking: + properties: + nodePort: + properties: + broadcast: + type: integer + broadcastSsl: + type: integer + cql: + type: integer + cqlSsl: + type: integer + type: object + type: object nodeSelector: additionalProperties: type: string diff --git a/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml b/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml index 41e50410e..702f4c8f3 100644 --- a/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml +++ b/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml @@ -92,6 +92,20 @@ spec: - serverSecretName type: object type: object + networking: + properties: + nodePort: + properties: + broadcast: + type: integer + broadcastSsl: + type: integer + cql: + type: integer + cqlSsl: + type: integer + type: object + type: object nodeSelector: additionalProperties: type: string diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go index e6e53aa72..6c0647968 100644 --- a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -221,11 +221,29 @@ type CassandraDatacenterSpec struct { // Cassandra users to bootstrap Users []CassandraUser `json:"users,omitempty"` + Networking *NetworkingConfig `json:"networking,omitempty"` + AdditionalSeeds []string `json:"additionalSeeds,omitempty"` Reaper *ReaperConfig `json:"reaper,omitempty"` } +type NetworkingConfig struct { + NodePort *NodePortConfig `json:"nodePort,omitempty"` +} + +type NodePortConfig struct { + Cql int `json:"cql,omitempty"` + CqlSsl int `json:"cqlSsl,omitempty"` + Broadcast int `json:"broadcast,omitempty"` + BroadcastSsl int `json:"broadcastSsl,omitempty"` +} + +// Is the NodePort service enabled? +func (dc *CassandraDatacenter) IsNodePortEnabled() bool { + return dc.Spec.Networking != nil && dc.Spec.Networking.NodePort != nil +} + type DseWorkloads struct { AnalyticsEnabled bool `json:"analyticsEnabled,omitempty"` GraphEnabled bool `json:"graphEnabled,omitempty"` @@ -479,10 +497,6 @@ func (dc *CassandraDatacenter) GetSeedServiceName() string { return dc.Spec.ClusterName + "-seed-service" } -func (dc *CassandraDatacenter) GetAdditionalSeedsServiceName() string { - return dc.Spec.ClusterName + "-" + dc.Name + fmt.Sprintf("-additional-seed-service") -} - func (dc *CassandraDatacenter) GetAllPodsServiceName() string { return dc.Spec.ClusterName + "-" + dc.Name + "-all-pods-service" } @@ -533,13 +547,28 @@ func (dc *CassandraDatacenter) GetConfigAsJSON() (string, error) { } } + cql := 0 + cqlSsl := 0 + broadcast := 0 + broadcastSsl := 0 + if dc.IsNodePortEnabled() { + cql = dc.Spec.Networking.NodePort.Cql + cqlSsl = dc.Spec.Networking.NodePort.CqlSsl + broadcast = dc.Spec.Networking.NodePort.Broadcast + broadcastSsl = dc.Spec.Networking.NodePort.BroadcastSsl + } + modelValues := serverconfig.GetModelValues( seeds, dc.Spec.ClusterName, dc.Name, graphEnabled, solrEnabled, - sparkEnabled) + sparkEnabled, + cql, + cqlSsl, + broadcast, + broadcastSsl) var modelBytes []byte @@ -569,13 +598,53 @@ func (dc *CassandraDatacenter) GetConfigAsJSON() (string, error) { return modelParsed.String(), nil } +// Gets the defined CQL port for NodePort. +// 0 will be returned if NodePort is not configured. +// The SSL port will be returned if it is defined, +// otherwise the normal CQL port will be used. +func (dc *CassandraDatacenter) GetNodePortCqlPort() int { + if !dc.IsNodePortEnabled() { + return 0 + } + + if dc.Spec.Networking.NodePort.CqlSsl != 0 { + return dc.Spec.Networking.NodePort.CqlSsl + } else if dc.Spec.Networking.NodePort.Cql != 0 { + return dc.Spec.Networking.NodePort.Cql + } else { + return 9042 + } +} + +// Gets the defined broadcast/intranode port for NodePort. +// 0 will be returned if NodePort is not configured. +// The SSL port will be returned if it is defined, +// otherwise the normal broadcast port will be used. +func (dc *CassandraDatacenter) GetNodePortBroadcastPort() int { + if !dc.IsNodePortEnabled() { + return 0 + } + + if dc.Spec.Networking.NodePort.BroadcastSsl != 0 { + return dc.Spec.Networking.NodePort.BroadcastSsl + } else if dc.Spec.Networking.NodePort.Broadcast != 0 { + return dc.Spec.Networking.NodePort.Broadcast + } else { + return 7000 + } +} + // GetContainerPorts will return the container ports for the pods in a statefulset based on the provided config func (dc *CassandraDatacenter) GetContainerPorts() ([]corev1.ContainerPort, error) { + + cqlPort := 9042 + broadcastPort := 7000 + ports := []corev1.ContainerPort{ { // Note: Port Names cannot be more than 15 characters Name: "native", - ContainerPort: 9042, + ContainerPort: int32(cqlPort), }, { Name: "inter-node-msg", @@ -583,7 +652,7 @@ func (dc *CassandraDatacenter) GetContainerPorts() ([]corev1.ContainerPort, erro }, { Name: "intra-node", - ContainerPort: 7000, + ContainerPort: int32(broadcastPort), }, { Name: "tls-intra-node", diff --git a/operator/pkg/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/operator/pkg/apis/cassandra/v1beta1/zz_generated.deepcopy.go index d964ee1d9..ad85a57ff 100644 --- a/operator/pkg/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/operator/pkg/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -120,6 +120,11 @@ func (in *CassandraDatacenterSpec) DeepCopyInto(out *CassandraDatacenterSpec) { *out = make([]CassandraUser, len(*in)) copy(*out, *in) } + if in.Networking != nil { + in, out := &in.Networking, &out.Networking + *out = new(NetworkingConfig) + (*in).DeepCopyInto(*out) + } if in.AdditionalSeeds != nil { in, out := &in.AdditionalSeeds, &out.AdditionalSeeds *out = make([]string, len(*in)) @@ -327,6 +332,43 @@ func (in *ManagementApiAuthManualConfig) DeepCopy() *ManagementApiAuthManualConf return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkingConfig) DeepCopyInto(out *NetworkingConfig) { + *out = *in + if in.NodePort != nil { + in, out := &in.NodePort, &out.NodePort + *out = new(NodePortConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkingConfig. +func (in *NetworkingConfig) DeepCopy() *NetworkingConfig { + if in == nil { + return nil + } + out := new(NetworkingConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePortConfig. +func (in *NodePortConfig) DeepCopy() *NodePortConfig { + if in == nil { + return nil + } + out := new(NodePortConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Rack) DeepCopyInto(out *Rack) { *out = *in From 177ebd0f99879a157e3db7498e2d2a99aeb25685 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Fri, 24 Jul 2020 14:09:48 -0500 Subject: [PATCH 02/16] added configgen port setting --- operator/pkg/serverconfig/configgen.go | 22 ++- operator/pkg/serverconfig/configgen_test.go | 140 ++++++++++++++------ 2 files changed, 118 insertions(+), 44 deletions(-) diff --git a/operator/pkg/serverconfig/configgen.go b/operator/pkg/serverconfig/configgen.go index 2a97a5420..e390603d7 100644 --- a/operator/pkg/serverconfig/configgen.go +++ b/operator/pkg/serverconfig/configgen.go @@ -17,7 +17,11 @@ func GetModelValues( dcName string, graphEnabled int, solrEnabled int, - sparkEnabled int) NodeConfig { + sparkEnabled int, + cqlPort int, + cqlSslPort int, + broadcastPort int, + broadcastSslPort int) NodeConfig { seedsString := strings.Join(seeds, ",") @@ -32,7 +36,21 @@ func GetModelValues( "graph-enabled": graphEnabled, "solr-enabled": solrEnabled, "spark-enabled": sparkEnabled, - }} + }, + "cassandra-yaml": NodeConfig{}, + } + + if cqlSslPort != 0 { + modelValues["cassandra-yaml"].(NodeConfig)["native_transport_port_ssl"] = cqlSslPort + } else if cqlPort != 0 { + modelValues["cassandra-yaml"].(NodeConfig)["native_transport_port"] = cqlPort + } + + if broadcastSslPort != 0 { + modelValues["cassandra-yaml"].(NodeConfig)["ssl_storage_port"] = broadcastSslPort + } else if broadcastPort != 0 { + modelValues["cassandra-yaml"].(NodeConfig)["storage_port"] = broadcastPort + } return modelValues } diff --git a/operator/pkg/serverconfig/configgen_test.go b/operator/pkg/serverconfig/configgen_test.go index 27656511a..88ed6fb07 100644 --- a/operator/pkg/serverconfig/configgen_test.go +++ b/operator/pkg/serverconfig/configgen_test.go @@ -10,12 +10,16 @@ import ( func TestGetModelValues(t *testing.T) { type args struct { - seeds []string - clusterName string - dcName string - graphEnabled int - solrEnabled int - sparkEnabled int + seeds []string + clusterName string + dcName string + graphEnabled int + solrEnabled int + sparkEnabled int + cqlPort int + cqlSslPort int + broadcastPort int + broadcastSslPort int } tests := []struct { name string @@ -25,12 +29,16 @@ func TestGetModelValues(t *testing.T) { { name: "Happy Path", args: args{ - seeds: []string{"seed0", "seed1", "seed2"}, - clusterName: "cluster-name", - dcName: "dc-name", - graphEnabled: 1, - solrEnabled: 0, - sparkEnabled: 0, + seeds: []string{"seed0", "seed1", "seed2"}, + clusterName: "cluster-name", + dcName: "dc-name", + graphEnabled: 1, + solrEnabled: 0, + sparkEnabled: 0, + cqlPort: 9042, + cqlSslPort: 0, + broadcastPort: 7000, + broadcastSslPort: 7000, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -42,17 +50,26 @@ func TestGetModelValues(t *testing.T) { "name": "dc-name", "solr-enabled": 0, "spark-enabled": 0, - }}, + }, + "cassandra-yaml": NodeConfig{ + "native_transport_port": 9042, + "ssl_storage_port": 7000, + }, + }, }, { name: "Empty seeds", args: args{ - seeds: []string{}, - clusterName: "cluster-name", - dcName: "dc-name", - graphEnabled: 0, - solrEnabled: 1, - sparkEnabled: 0, + seeds: []string{}, + clusterName: "cluster-name", + dcName: "dc-name", + graphEnabled: 0, + solrEnabled: 1, + sparkEnabled: 0, + cqlPort: 9042, + cqlSslPort: 9142, + broadcastPort: 7000, + broadcastSslPort: 0, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -64,17 +81,26 @@ func TestGetModelValues(t *testing.T) { "name": "dc-name", "solr-enabled": 1, "spark-enabled": 0, - }}, + }, + "cassandra-yaml": NodeConfig{ + "native_transport_port_ssl": 9142, + "storage_port": 7000, + }, + }, }, { name: "Missing cluster name", args: args{ - seeds: []string{"seed0", "seed1", "seed2"}, - clusterName: "", - dcName: "dc-name", - graphEnabled: 1, - solrEnabled: 1, - sparkEnabled: 1, + seeds: []string{"seed0", "seed1", "seed2"}, + clusterName: "", + dcName: "dc-name", + graphEnabled: 1, + solrEnabled: 1, + sparkEnabled: 1, + cqlPort: 9042, + cqlSslPort: 0, + broadcastPort: 7200, + broadcastSslPort: 7300, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -86,17 +112,26 @@ func TestGetModelValues(t *testing.T) { "name": "dc-name", "solr-enabled": 1, "spark-enabled": 1, - }}, + }, + "cassandra-yaml": NodeConfig{ + "native_transport_port": 9042, + "ssl_storage_port": 7300, + }, + }, }, { name: "Missing dc name", args: args{ - seeds: []string{"seed0", "seed1", "seed2"}, - clusterName: "cluster-name", - dcName: "", - graphEnabled: 0, - solrEnabled: 0, - sparkEnabled: 1, + seeds: []string{"seed0", "seed1", "seed2"}, + clusterName: "cluster-name", + dcName: "", + graphEnabled: 0, + solrEnabled: 0, + sparkEnabled: 1, + cqlPort: 9142, + cqlSslPort: 0, + broadcastPort: 7000, + broadcastSslPort: 0, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -108,17 +143,26 @@ func TestGetModelValues(t *testing.T) { "name": "", "solr-enabled": 0, "spark-enabled": 1, - }}, + }, + "cassandra-yaml": NodeConfig{ + "native_transport_port": 9142, + "storage_port": 7000, + }, + }, }, { name: "Empty args", args: args{ - seeds: nil, - clusterName: "", - dcName: "", - graphEnabled: 0, - solrEnabled: 0, - sparkEnabled: 0, + seeds: nil, + clusterName: "", + dcName: "", + graphEnabled: 0, + solrEnabled: 0, + sparkEnabled: 0, + cqlPort: 0, + cqlSslPort: 0, + broadcastPort: 0, + broadcastSslPort: 0, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -130,12 +174,24 @@ func TestGetModelValues(t *testing.T) { "name": "", "solr-enabled": 0, "spark-enabled": 0, - }}, + }, + "cassandra-yaml": NodeConfig{}, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := GetModelValues(tt.args.seeds, tt.args.clusterName, tt.args.dcName, tt.args.graphEnabled, tt.args.solrEnabled, tt.args.sparkEnabled); !reflect.DeepEqual(got, tt.want) { + if got := GetModelValues( + tt.args.seeds, + tt.args.clusterName, + tt.args.dcName, + tt.args.graphEnabled, + tt.args.solrEnabled, + tt.args.sparkEnabled, + tt.args.cqlPort, + tt.args.cqlSslPort, + tt.args.broadcastPort, + tt.args.broadcastSslPort); !reflect.DeepEqual(got, tt.want) { t.Errorf("GetModelValues() = %v, want %v", got, tt.want) } }) From f714499aed62d8d902a8ba4ce15c975fca5a0aef Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Fri, 24 Jul 2020 14:28:54 -0500 Subject: [PATCH 03/16] Passes HOST_IP to configbuilder --- operator/pkg/reconciliation/constructor.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/operator/pkg/reconciliation/constructor.go b/operator/pkg/reconciliation/constructor.go index c6a87ab56..6f9454159 100644 --- a/operator/pkg/reconciliation/constructor.go +++ b/operator/pkg/reconciliation/constructor.go @@ -504,6 +504,12 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string) ([]corev1 } serverCfg.VolumeMounts = []corev1.VolumeMount{serverCfgMount} + // Convert the bool to a string for the env var setting + useHostIpForBroadcast := "false" + if dc.IsNodePortEnabled() { + useHostIpForBroadcast = "true" + } + configData, err := dc.GetConfigAsJSON() if err != nil { return nil, err @@ -512,6 +518,8 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string) ([]corev1 serverCfg.Env = []corev1.EnvVar{ {Name: "CONFIG_FILE_DATA", Value: configData}, {Name: "POD_IP", ValueFrom: selectorFromFieldPath("status.podIP")}, + {Name: "HOST_IP", ValueFrom: selectorFromFieldPath("status.hostIP")}, + {Name: "USE_HOST_IP_FOR_BROADCAST", Value: useHostIpForBroadcast}, {Name: "RACK_NAME", Value: rackName}, {Name: "PRODUCT_VERSION", Value: serverVersion}, {Name: "PRODUCT_NAME", Value: dc.Spec.ServerType}, From cd3408822ecb43d25d10f7ae2e968557b61878cd Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Fri, 24 Jul 2020 14:34:23 -0500 Subject: [PATCH 04/16] replaced accidentally removed function --- .../pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go index 6c0647968..fa054afa5 100644 --- a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -497,6 +497,10 @@ func (dc *CassandraDatacenter) GetSeedServiceName() string { return dc.Spec.ClusterName + "-seed-service" } +func (dc *CassandraDatacenter) GetAdditionalSeedsServiceName() string { + return dc.Spec.ClusterName + "-" + dc.Name + fmt.Sprintf("-additional-seed-service") +} + func (dc *CassandraDatacenter) GetAllPodsServiceName() string { return dc.Spec.ClusterName + "-" + dc.Name + "-all-pods-service" } From 111a036df68fe182e91d3d1c208dd8f9e8c99dfe Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Fri, 24 Jul 2020 15:28:57 -0500 Subject: [PATCH 05/16] WIP - create nodeport service --- .../v1beta1/cassandradatacenter_types.go | 4 ++ operator/pkg/reconciliation/constructor.go | 39 +++++++++++++++++++ .../pkg/reconciliation/reconcile_services.go | 5 +++ 3 files changed, 48 insertions(+) diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go index fa054afa5..ca1138578 100644 --- a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -509,6 +509,10 @@ func (dc *CassandraDatacenter) GetDatacenterServiceName() string { return dc.Spec.ClusterName + "-" + dc.Name + "-service" } +func (dc *CassandraDatacenter) GetNodePortServiceName() string { + return dc.Spec.ClusterName + "-" + dc.Name + "-node-port-service" +} + func (dc *CassandraDatacenter) ShouldGenerateSuperuserSecret() bool { return len(dc.Spec.SuperuserSecretName) == 0 } diff --git a/operator/pkg/reconciliation/constructor.go b/operator/pkg/reconciliation/constructor.go index 6f9454159..8800ce2c0 100644 --- a/operator/pkg/reconciliation/constructor.go +++ b/operator/pkg/reconciliation/constructor.go @@ -122,6 +122,45 @@ func newEndpointsForAdditionalSeeds(dc *api.CassandraDatacenter) *corev1.Endpoin return &endpoints } +// newNodePortServiceForCassandraDatacenter creates a headless service owned by the CassandraDatacenter, +// that preserves the client source IPs +func newNodePortServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Service { + service := makeGenericHeadlessService(dc) + service.ObjectMeta.Name = dc.GetSeedServiceName() + + service.Spec.Type = "NodePort" + // Note: ClusterIp = "None" is not valid for NodePort + service.Spec.ClusterIP = "" + service.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal + + cqlPort := dc.GetNodePortCqlPort() + broadcastPort := dc.GetNodePortBroadcastPort() + + service.Spec.Ports = []corev1.ServicePort{ + // Note: Port Names cannot be more than 15 characters + { + Name: "broadcast", + Port: int32(broadcastPort), + NodePort: int32(broadcastPort), + TargetPort: intstr.FromInt(broadcastPort), + }, + { + Name: "native", + Port: int32(cqlPort), + NodePort: int32(cqlPort), + TargetPort: intstr.FromInt(cqlPort), + }, + { + Name: "mgmt-api", + Port: 8080, + NodePort: int32(8080), + TargetPort: intstr.FromInt(8080), + }, + } + + return service +} + // newAllPodsServiceForCassandraDatacenter creates a headless service owned by the CassandraDatacenter, // which covers all server pods in the datacenter, whether they are ready or not func newAllPodsServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Service { diff --git a/operator/pkg/reconciliation/reconcile_services.go b/operator/pkg/reconciliation/reconcile_services.go index 5e7db3d3a..ad2f8bd59 100644 --- a/operator/pkg/reconciliation/reconcile_services.go +++ b/operator/pkg/reconciliation/reconcile_services.go @@ -66,6 +66,11 @@ func (rc *ReconciliationContext) CheckHeadlessServices() result.ReconcileResult services = append(services, additionalSeedService) } + if len(dc.IsNodePortEnabled()) { + nodePortService := newNodePortServiceForCassandraDatacenter(dc) + services = append(services, nodePortService) + } + createNeeded := []*corev1.Service{} for idx := range services { From 88765b2c25515c64e234e463dbb8c0cee5250541 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Fri, 24 Jul 2020 16:01:10 -0500 Subject: [PATCH 06/16] WIP - created nodeport test --- .../pkg/reconciliation/reconcile_services.go | 2 +- .../nodeport_service_suite_test.go | 70 +++++++++++++++++++ tests/testdata/nodeport-service-dc.yaml | 33 +++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 tests/nodeport_service/nodeport_service_suite_test.go create mode 100644 tests/testdata/nodeport-service-dc.yaml diff --git a/operator/pkg/reconciliation/reconcile_services.go b/operator/pkg/reconciliation/reconcile_services.go index ad2f8bd59..f722b8e9e 100644 --- a/operator/pkg/reconciliation/reconcile_services.go +++ b/operator/pkg/reconciliation/reconcile_services.go @@ -66,7 +66,7 @@ func (rc *ReconciliationContext) CheckHeadlessServices() result.ReconcileResult services = append(services, additionalSeedService) } - if len(dc.IsNodePortEnabled()) { + if dc.IsNodePortEnabled() { nodePortService := newNodePortServiceForCassandraDatacenter(dc) services = append(services, nodePortService) } diff --git a/tests/nodeport_service/nodeport_service_suite_test.go b/tests/nodeport_service/nodeport_service_suite_test.go new file mode 100644 index 000000000..64622912f --- /dev/null +++ b/tests/nodeport_service/nodeport_service_suite_test.go @@ -0,0 +1,70 @@ +// Copyright DataStax, Inc. +// Please see the included license file for details. + +package nodeport_service + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + + ginkgo_util "github.com/datastax/cass-operator/mage/ginkgo" + "github.com/datastax/cass-operator/mage/kubectl" +) + +var ( + testName = "NodePort Service" + namespace = "test-node-port-service" + dcName = "dc1" + dcYaml = "../testdata/nodeport-service-dc.yaml" + operatorYaml = "../testdata/operator.yaml" + dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName) + dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) + // additionalSeedServiceResource = "services/cluster1-dc1-additional-seed-service" + //additionalSeedEndpointResource = "endpoints/cluster1-dc1-additional-seed-service" + ns = ginkgo_util.NewWrapper(testName, namespace) +) + +func TestLifecycle(t *testing.T) { + AfterSuite(func() { + logPath := fmt.Sprintf("%s/aftersuite", ns.LogDir) + kubectl.DumpAllLogs(logPath).ExecV() + + fmt.Printf("\n\tPost-run logs dumped at: %s\n\n", logPath) + ns.Terminate() + }) + + RegisterFailHandler(Fail) + RunSpecs(t, testName) +} + +var _ = Describe(testName, func() { + Context("when in a new cluster", func() { + Specify("the operator can properly create a nodeport service", func() { + var step string + var k kubectl.KCmd + + By("creating a namespace") + err := kubectl.CreateNamespace(namespace).ExecV() + Expect(err).ToNot(HaveOccurred()) + + step = "setting up cass-operator resources via helm chart" + ns.HelmInstall("../../charts/cass-operator-chart") + + ns.WaitForOperatorReady() + + step = "creating a datacenter resource with a nodeport service" + k = kubectl.ApplyFiles(dcYaml) + ns.ExecAndLog(step, k) + + ns.WaitForDatacenterReady(dcName) + }) + }) +}) diff --git a/tests/testdata/nodeport-service-dc.yaml b/tests/testdata/nodeport-service-dc.yaml new file mode 100644 index 000000000..96182fcdd --- /dev/null +++ b/tests/testdata/nodeport-service-dc.yaml @@ -0,0 +1,33 @@ +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: dc1 +spec: + clusterName: cluster1 + serverType: dse + serverVersion: "6.8.1" + managementApiAuth: + insecure: {} + networking: + nodePort: + cqlSsl: 12121 + broadcast: 12122 + size: 2 + storageConfig: + cassandraDataVolumeClaimSpec: + storageClassName: server-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + racks: + - name: r1 + - name: r2 + config: + jvm-server-options: + initial_heap_size: "800m" + max_heap_size: "800m" + cassandra-yaml: + file_cache_size_in_mb: 100 + memtable_space_in_mb: 100 From bb2c9d4501a77fc51efbed977a510ccf48ff8680 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Fri, 24 Jul 2020 16:54:56 -0500 Subject: [PATCH 07/16] WIP - test fixes --- operator/pkg/reconciliation/constructor.go | 2 +- tests/nodeport_service/nodeport_service_suite_test.go | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/operator/pkg/reconciliation/constructor.go b/operator/pkg/reconciliation/constructor.go index 8800ce2c0..6a4f68572 100644 --- a/operator/pkg/reconciliation/constructor.go +++ b/operator/pkg/reconciliation/constructor.go @@ -126,7 +126,7 @@ func newEndpointsForAdditionalSeeds(dc *api.CassandraDatacenter) *corev1.Endpoin // that preserves the client source IPs func newNodePortServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Service { service := makeGenericHeadlessService(dc) - service.ObjectMeta.Name = dc.GetSeedServiceName() + service.ObjectMeta.Name = dc.GetNodePortServiceName() service.Spec.Type = "NodePort" // Note: ClusterIp = "None" is not valid for NodePort diff --git a/tests/nodeport_service/nodeport_service_suite_test.go b/tests/nodeport_service/nodeport_service_suite_test.go index 64622912f..6b279a8be 100644 --- a/tests/nodeport_service/nodeport_service_suite_test.go +++ b/tests/nodeport_service/nodeport_service_suite_test.go @@ -4,16 +4,13 @@ package nodeport_service import ( - "encoding/json" "fmt" - "regexp" - "sort" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" + // corev1 "k8s.io/api/core/v1" ginkgo_util "github.com/datastax/cass-operator/mage/ginkgo" "github.com/datastax/cass-operator/mage/kubectl" @@ -25,8 +22,8 @@ var ( dcName = "dc1" dcYaml = "../testdata/nodeport-service-dc.yaml" operatorYaml = "../testdata/operator.yaml" - dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName) - dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) + // dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName) + // dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) // additionalSeedServiceResource = "services/cluster1-dc1-additional-seed-service" //additionalSeedEndpointResource = "endpoints/cluster1-dc1-additional-seed-service" ns = ginkgo_util.NewWrapper(testName, namespace) From 12729e4dce957b60130508a62a20e34df77bef71 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Mon, 27 Jul 2020 12:44:50 -0500 Subject: [PATCH 08/16] fixed a port and introduced a couple of test helper methods --- mage/ginkgo/lib.go | 13 ++++- operator/pkg/reconciliation/constructor.go | 6 --- .../nodeport_service_suite_test.go | 54 +++++++++++++++---- tests/testdata/nodeport-service-dc.yaml | 4 +- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/mage/ginkgo/lib.go b/mage/ginkgo/lib.go index 331d40a9b..95be0cff3 100644 --- a/mage/ginkgo/lib.go +++ b/mage/ginkgo/lib.go @@ -214,7 +214,6 @@ func (ns *NsWrapper) WaitForOutputContainsAndLog(description string, kcmd kubect Expect(execErr).ToNot(HaveOccurred()) } - func (ns *NsWrapper) WaitForDatacenterCondition(dcName string, conditionType string, value string) { step := fmt.Sprintf("checking that dc condition %s has value %s", conditionType, value) json := fmt.Sprintf("jsonpath={.status.conditions[?(.type=='%s')].status}", conditionType) @@ -223,7 +222,6 @@ func (ns *NsWrapper) WaitForDatacenterCondition(dcName string, conditionType str ns.WaitForOutputAndLog(step, k, value, 600) } - func (ns *NsWrapper) WaitForDatacenterToHaveNoPods(dcName string) { step := "checking that no dc pods remain" json := "jsonpath={.items}" @@ -369,3 +367,14 @@ func (ns NsWrapper) HelmInstall(chartPath string) { err := helm_util.Install(chartPath, "cass-operator", ns.Namespace, overrides) mageutil.PanicOnError(err) } + +func (ns NsWrapper) ExpectKeyValue(m map[string]interface{}, key string, expectedValue string) { + Expect(m[key]).To(Equal(expectedValue), "Expected %s %s to be %s", key, m[key], expectedValue) +} + +// Compare all key/values from an expected map to an actual map +func (ns NsWrapper) ExpectKeyValues(actual map[string]interface{}, expected map[string]string) { + for key := range expected { + ns.ExpectKeyValue(actual, key, expected[key]) + } +} diff --git a/operator/pkg/reconciliation/constructor.go b/operator/pkg/reconciliation/constructor.go index 6a4f68572..f635bd157 100644 --- a/operator/pkg/reconciliation/constructor.go +++ b/operator/pkg/reconciliation/constructor.go @@ -150,12 +150,6 @@ func newNodePortServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *core NodePort: int32(cqlPort), TargetPort: intstr.FromInt(cqlPort), }, - { - Name: "mgmt-api", - Port: 8080, - NodePort: int32(8080), - TargetPort: intstr.FromInt(8080), - }, } return service diff --git a/tests/nodeport_service/nodeport_service_suite_test.go b/tests/nodeport_service/nodeport_service_suite_test.go index 6b279a8be..53609f95b 100644 --- a/tests/nodeport_service/nodeport_service_suite_test.go +++ b/tests/nodeport_service/nodeport_service_suite_test.go @@ -4,6 +4,7 @@ package nodeport_service import ( + "encoding/json" "fmt" "testing" @@ -17,16 +18,13 @@ import ( ) var ( - testName = "NodePort Service" - namespace = "test-node-port-service" - dcName = "dc1" - dcYaml = "../testdata/nodeport-service-dc.yaml" - operatorYaml = "../testdata/operator.yaml" - // dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName) - // dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) - // additionalSeedServiceResource = "services/cluster1-dc1-additional-seed-service" - //additionalSeedEndpointResource = "endpoints/cluster1-dc1-additional-seed-service" - ns = ginkgo_util.NewWrapper(testName, namespace) + testName = "NodePort Service" + namespace = "test-node-port-service" + dcName = "dc1" + dcYaml = "../testdata/nodeport-service-dc.yaml" + operatorYaml = "../testdata/operator.yaml" + nodePortServiceResource = "services/cluster1-dc1-node-port-service" + ns = ginkgo_util.NewWrapper(testName, namespace) ) func TestLifecycle(t *testing.T) { @@ -42,6 +40,40 @@ func TestLifecycle(t *testing.T) { RunSpecs(t, testName) } +func checkNodePortService() { + // Check the service + + k := kubectl.Get(nodePortServiceResource).FormatOutput("json") + output := ns.OutputPanic(k) + data := map[string]interface{}{} + err := json.Unmarshal([]byte(output), &data) + Expect(err).ToNot(HaveOccurred()) + + err = json.Unmarshal([]byte(output), &data) + + spec := data["spec"].(map[string]interface{}) + policy := spec["externalTrafficPolicy"].(string) + Expect(policy).To(Equal("Local"), "Expected externalTrafficPolicy %s to be Local", policy) + + portData := spec["ports"].([]interface{}) + port0 := portData[0].(map[string]interface{}) + port1 := portData[1].(map[string]interface{}) + + ns.ExpectKeyValues(port0, map[string]string{ + "name": "broadcast", + "nodePort": "30002", + "port": "30002", + "targetPort": "30002", + }) + + ns.ExpectKeyValues(port1, map[string]string{ + "name": "native", + "nodePort": "30001", + "port": "30001", + "targetPort": "30001", + }) +} + var _ = Describe(testName, func() { Context("when in a new cluster", func() { Specify("the operator can properly create a nodeport service", func() { @@ -62,6 +94,8 @@ var _ = Describe(testName, func() { ns.ExecAndLog(step, k) ns.WaitForDatacenterReady(dcName) + + checkNodePortService() }) }) }) diff --git a/tests/testdata/nodeport-service-dc.yaml b/tests/testdata/nodeport-service-dc.yaml index 96182fcdd..374f3efaf 100644 --- a/tests/testdata/nodeport-service-dc.yaml +++ b/tests/testdata/nodeport-service-dc.yaml @@ -10,8 +10,8 @@ spec: insecure: {} networking: nodePort: - cqlSsl: 12121 - broadcast: 12122 + cqlSsl: 30001 + broadcast: 30002 size: 2 storageConfig: cassandraDataVolumeClaimSpec: From 2c0e453529e01ca265d38db69074951c5ab67e02 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Mon, 27 Jul 2020 16:49:13 -0500 Subject: [PATCH 09/16] do not set a native port in the normal service if the nodeport service is enabled --- operator/pkg/reconciliation/constructor.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/operator/pkg/reconciliation/constructor.go b/operator/pkg/reconciliation/constructor.go index f635bd157..f4a037677 100644 --- a/operator/pkg/reconciliation/constructor.go +++ b/operator/pkg/reconciliation/constructor.go @@ -33,15 +33,20 @@ func newServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Servi service := makeGenericHeadlessService(dc) service.ObjectMeta.Name = svcName service.Spec.Ports = []corev1.ServicePort{ - // Note: Port Names cannot be more than 15 characters - { - Name: "native", Port: 9042, TargetPort: intstr.FromInt(9042), - }, { Name: "mgmt-api", Port: 8080, TargetPort: intstr.FromInt(8080), }, } + // If NodePort is enabled, it will take control of the native port + if !dc.IsNodePortEnabled() { + service.Spec.Ports = append(service.Spec.Ports, + corev1.ServicePort{ + Name: "native", Port: 9042, TargetPort: intstr.FromInt(9042), + }, + ) + } + addHashAnnotation(service) return service From 9e91c5c2507ba62d7c85e3866b680d16439c3eaa Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Tue, 28 Jul 2020 11:36:48 -0500 Subject: [PATCH 10/16] use cql port instead of cqlSsl port in the test --- tests/testdata/nodeport-service-dc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata/nodeport-service-dc.yaml b/tests/testdata/nodeport-service-dc.yaml index 374f3efaf..62ae57361 100644 --- a/tests/testdata/nodeport-service-dc.yaml +++ b/tests/testdata/nodeport-service-dc.yaml @@ -10,7 +10,7 @@ spec: insecure: {} networking: nodePort: - cqlSsl: 30001 + cql: 30001 broadcast: 30002 size: 2 storageConfig: From 7bccdff2e229ce0d1aa65cd5802b8587e7248972 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Tue, 28 Jul 2020 13:53:24 -0500 Subject: [PATCH 11/16] preserve ClusterIP of existing services --- operator/pkg/reconciliation/reconcile_services.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/operator/pkg/reconciliation/reconcile_services.go b/operator/pkg/reconciliation/reconcile_services.go index f722b8e9e..67f213f7f 100644 --- a/operator/pkg/reconciliation/reconcile_services.go +++ b/operator/pkg/reconciliation/reconcile_services.go @@ -107,6 +107,11 @@ func (rc *ReconciliationContext) CheckHeadlessServices() result.ReconcileResult desiredSvc.Labels = utils.MergeMap(map[string]string{}, currentService.Labels, desiredSvc.Labels) desiredSvc.Annotations = utils.MergeMap(map[string]string{}, currentService.Annotations, desiredSvc.Annotations) + // ClusterIP may have been updated for the NodePort service + // so we need to preserve it. Copying should not break any of + // the other services either. + desiredSvc.Spec.ClusterIP = currentService.Spec.ClusterIP + logger.Info("Updating service", "service", currentService, "desired", desiredSvc) From 9cb18522c59143e61216c5cd3ae9eff5862a637f Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Tue, 28 Jul 2020 14:28:03 -0500 Subject: [PATCH 12/16] the test runs --- mage/ginkgo/lib.go | 15 ++++++++++++++- .../nodeport_service_suite_test.go | 13 +++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/mage/ginkgo/lib.go b/mage/ginkgo/lib.go index 95be0cff3..9deceb9a7 100644 --- a/mage/ginkgo/lib.go +++ b/mage/ginkgo/lib.go @@ -368,8 +368,21 @@ func (ns NsWrapper) HelmInstall(chartPath string) { mageutil.PanicOnError(err) } +// Note that the actual value will be cast to a string before the comparison with the expectedValue func (ns NsWrapper) ExpectKeyValue(m map[string]interface{}, key string, expectedValue string) { - Expect(m[key]).To(Equal(expectedValue), "Expected %s %s to be %s", key, m[key], expectedValue) + actualValue, ok := m[key].(string) + if !ok { + // Note: floats will end up as strings with six decimal points + // example: "12.000000" + tryFloat64, ok := m[key].(float64) + if !ok { + msg := fmt.Sprintf("Actual value for key %s is not expected type", key) + err := fmt.Errorf(msg) + Expect(err).ToNot(HaveOccurred()) + } + actualValue = fmt.Sprintf("%f", tryFloat64) + } + Expect(actualValue).To(Equal(expectedValue), "Expected %s %s to be %s", key, m[key], expectedValue) } // Compare all key/values from an expected map to an actual map diff --git a/tests/nodeport_service/nodeport_service_suite_test.go b/tests/nodeport_service/nodeport_service_suite_test.go index 53609f95b..7c12eba9e 100644 --- a/tests/nodeport_service/nodeport_service_suite_test.go +++ b/tests/nodeport_service/nodeport_service_suite_test.go @@ -59,18 +59,19 @@ func checkNodePortService() { port0 := portData[0].(map[string]interface{}) port1 := portData[1].(map[string]interface{}) + // for some reason, k8s is giving the port numbers back as floats ns.ExpectKeyValues(port0, map[string]string{ "name": "broadcast", - "nodePort": "30002", - "port": "30002", - "targetPort": "30002", + "nodePort": "30002.000000", + "port": "30002.000000", + "targetPort": "30002.000000", }) ns.ExpectKeyValues(port1, map[string]string{ "name": "native", - "nodePort": "30001", - "port": "30001", - "targetPort": "30001", + "nodePort": "30001.000000", + "port": "30001.000000", + "targetPort": "30001.000000", }) } From 15004dd5c4385728d48dcd03b5f7712d344dfc56 Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Tue, 28 Jul 2020 14:42:29 -0500 Subject: [PATCH 13/16] updated the user doc for nodeport --- docs/user/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/user/README.md b/docs/user/README.md index 1c9bd2c21..48d46380a 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -363,6 +363,24 @@ spec: serverImage: private-docker-registry.example.com/dse-img/dse:5f6e7d8c ``` +## Configuring a NodePort service + +A NodePort service may be requested by setting the following fields: + + networking: + nodePort: + cql: 30001 + broadcast: 30002 + +The SSL versions of the ports may be requested: + + networking: + nodePort: + cqlSql: 30010 + broadcastSql: 30020 + +If any of the nodePort fields have been configured then a NodePort service will be created that routes from the specified external port to the identically numbered internal port. Cassandra will be configured to listen on the specified ports. Additionally, the standard service will no longer define a "native" port. + # Using Your Cluster ## Connecting from inside the Kubernetes cluster From 50bc3e52cc98d758d61b94fe8e00b3dddea609ff Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Tue, 28 Jul 2020 15:02:32 -0500 Subject: [PATCH 14/16] Port names now use SSL instead of Ssl --- .../templates/customresourcedefinition.yaml | 4 +-- docs/user/README.md | 4 +-- ...datastax.com_cassandradatacenters_crd.yaml | 4 +-- .../v1beta1/cassandradatacenter_types.go | 24 ++++++++-------- operator/pkg/serverconfig/configgen.go | 12 ++++---- operator/pkg/serverconfig/configgen_test.go | 28 +++++++++---------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/charts/cass-operator-chart/templates/customresourcedefinition.yaml b/charts/cass-operator-chart/templates/customresourcedefinition.yaml index 512187a4b..4b4a507d1 100644 --- a/charts/cass-operator-chart/templates/customresourcedefinition.yaml +++ b/charts/cass-operator-chart/templates/customresourcedefinition.yaml @@ -98,11 +98,11 @@ spec: properties: broadcast: type: integer - broadcastSsl: + broadcastSSL: type: integer cql: type: integer - cqlSsl: + cqlSSL: type: integer type: object type: object diff --git a/docs/user/README.md b/docs/user/README.md index 48d46380a..2d211304a 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -376,8 +376,8 @@ The SSL versions of the ports may be requested: networking: nodePort: - cqlSql: 30010 - broadcastSql: 30020 + cqlSSL: 30010 + broadcastSSL: 30020 If any of the nodePort fields have been configured then a NodePort service will be created that routes from the specified external port to the identically numbered internal port. Cassandra will be configured to listen on the specified ports. Additionally, the standard service will no longer define a "native" port. diff --git a/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml b/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml index 702f4c8f3..dacacdda5 100644 --- a/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml +++ b/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml @@ -98,11 +98,11 @@ spec: properties: broadcast: type: integer - broadcastSsl: + broadcastSSL: type: integer cql: type: integer - cqlSsl: + cqlSSL: type: integer type: object type: object diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go index ca1138578..189c27f6a 100644 --- a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -234,9 +234,9 @@ type NetworkingConfig struct { type NodePortConfig struct { Cql int `json:"cql,omitempty"` - CqlSsl int `json:"cqlSsl,omitempty"` + CqlSSL int `json:"cqlSSL,omitempty"` Broadcast int `json:"broadcast,omitempty"` - BroadcastSsl int `json:"broadcastSsl,omitempty"` + BroadcastSSL int `json:"broadcastSSL,omitempty"` } // Is the NodePort service enabled? @@ -556,14 +556,14 @@ func (dc *CassandraDatacenter) GetConfigAsJSON() (string, error) { } cql := 0 - cqlSsl := 0 + cqlSSL := 0 broadcast := 0 - broadcastSsl := 0 + broadcastSSL := 0 if dc.IsNodePortEnabled() { cql = dc.Spec.Networking.NodePort.Cql - cqlSsl = dc.Spec.Networking.NodePort.CqlSsl + cqlSSL = dc.Spec.Networking.NodePort.CqlSSL broadcast = dc.Spec.Networking.NodePort.Broadcast - broadcastSsl = dc.Spec.Networking.NodePort.BroadcastSsl + broadcastSSL = dc.Spec.Networking.NodePort.BroadcastSSL } modelValues := serverconfig.GetModelValues( @@ -574,9 +574,9 @@ func (dc *CassandraDatacenter) GetConfigAsJSON() (string, error) { solrEnabled, sparkEnabled, cql, - cqlSsl, + cqlSSL, broadcast, - broadcastSsl) + broadcastSSL) var modelBytes []byte @@ -615,8 +615,8 @@ func (dc *CassandraDatacenter) GetNodePortCqlPort() int { return 0 } - if dc.Spec.Networking.NodePort.CqlSsl != 0 { - return dc.Spec.Networking.NodePort.CqlSsl + if dc.Spec.Networking.NodePort.CqlSSL != 0 { + return dc.Spec.Networking.NodePort.CqlSSL } else if dc.Spec.Networking.NodePort.Cql != 0 { return dc.Spec.Networking.NodePort.Cql } else { @@ -633,8 +633,8 @@ func (dc *CassandraDatacenter) GetNodePortBroadcastPort() int { return 0 } - if dc.Spec.Networking.NodePort.BroadcastSsl != 0 { - return dc.Spec.Networking.NodePort.BroadcastSsl + if dc.Spec.Networking.NodePort.BroadcastSSL != 0 { + return dc.Spec.Networking.NodePort.BroadcastSSL } else if dc.Spec.Networking.NodePort.Broadcast != 0 { return dc.Spec.Networking.NodePort.Broadcast } else { diff --git a/operator/pkg/serverconfig/configgen.go b/operator/pkg/serverconfig/configgen.go index e390603d7..9de10d34c 100644 --- a/operator/pkg/serverconfig/configgen.go +++ b/operator/pkg/serverconfig/configgen.go @@ -19,9 +19,9 @@ func GetModelValues( solrEnabled int, sparkEnabled int, cqlPort int, - cqlSslPort int, + cqlSSLPort int, broadcastPort int, - broadcastSslPort int) NodeConfig { + broadcastSSLPort int) NodeConfig { seedsString := strings.Join(seeds, ",") @@ -40,14 +40,14 @@ func GetModelValues( "cassandra-yaml": NodeConfig{}, } - if cqlSslPort != 0 { - modelValues["cassandra-yaml"].(NodeConfig)["native_transport_port_ssl"] = cqlSslPort + if cqlSSLPort != 0 { + modelValues["cassandra-yaml"].(NodeConfig)["native_transport_port_ssl"] = cqlSSLPort } else if cqlPort != 0 { modelValues["cassandra-yaml"].(NodeConfig)["native_transport_port"] = cqlPort } - if broadcastSslPort != 0 { - modelValues["cassandra-yaml"].(NodeConfig)["ssl_storage_port"] = broadcastSslPort + if broadcastSSLPort != 0 { + modelValues["cassandra-yaml"].(NodeConfig)["ssl_storage_port"] = broadcastSSLPort } else if broadcastPort != 0 { modelValues["cassandra-yaml"].(NodeConfig)["storage_port"] = broadcastPort } diff --git a/operator/pkg/serverconfig/configgen_test.go b/operator/pkg/serverconfig/configgen_test.go index 88ed6fb07..6bbd8deff 100644 --- a/operator/pkg/serverconfig/configgen_test.go +++ b/operator/pkg/serverconfig/configgen_test.go @@ -17,9 +17,9 @@ func TestGetModelValues(t *testing.T) { solrEnabled int sparkEnabled int cqlPort int - cqlSslPort int + cqlSSLPort int broadcastPort int - broadcastSslPort int + broadcastSSLPort int } tests := []struct { name string @@ -36,9 +36,9 @@ func TestGetModelValues(t *testing.T) { solrEnabled: 0, sparkEnabled: 0, cqlPort: 9042, - cqlSslPort: 0, + cqlSSLPort: 0, broadcastPort: 7000, - broadcastSslPort: 7000, + broadcastSSLPort: 7000, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -67,9 +67,9 @@ func TestGetModelValues(t *testing.T) { solrEnabled: 1, sparkEnabled: 0, cqlPort: 9042, - cqlSslPort: 9142, + cqlSSLPort: 9142, broadcastPort: 7000, - broadcastSslPort: 0, + broadcastSSLPort: 0, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -98,9 +98,9 @@ func TestGetModelValues(t *testing.T) { solrEnabled: 1, sparkEnabled: 1, cqlPort: 9042, - cqlSslPort: 0, + cqlSSLPort: 0, broadcastPort: 7200, - broadcastSslPort: 7300, + broadcastSSLPort: 7300, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -129,9 +129,9 @@ func TestGetModelValues(t *testing.T) { solrEnabled: 0, sparkEnabled: 1, cqlPort: 9142, - cqlSslPort: 0, + cqlSSLPort: 0, broadcastPort: 7000, - broadcastSslPort: 0, + broadcastSSLPort: 0, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -160,9 +160,9 @@ func TestGetModelValues(t *testing.T) { solrEnabled: 0, sparkEnabled: 0, cqlPort: 0, - cqlSslPort: 0, + cqlSSLPort: 0, broadcastPort: 0, - broadcastSslPort: 0, + broadcastSSLPort: 0, }, want: NodeConfig{ "cluster-info": NodeConfig{ @@ -189,9 +189,9 @@ func TestGetModelValues(t *testing.T) { tt.args.solrEnabled, tt.args.sparkEnabled, tt.args.cqlPort, - tt.args.cqlSslPort, + tt.args.cqlSSLPort, tt.args.broadcastPort, - tt.args.broadcastSslPort); !reflect.DeepEqual(got, tt.want) { + tt.args.broadcastSSLPort); !reflect.DeepEqual(got, tt.want) { t.Errorf("GetModelValues() = %v, want %v", got, tt.want) } }) From 8848f8145fbc6753371450afe5adf4f1c7e6c7dd Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Tue, 28 Jul 2020 15:17:24 -0500 Subject: [PATCH 15/16] Introduced DefaultCqlPort and DefaultBroadcastPort --- .../cassandra/v1beta1/cassandradatacenter_types.go | 12 ++++++++---- .../v1beta1/cassandradatacenter_types_test.go | 12 ++++++------ operator/pkg/reconciliation/constructor.go | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go index 189c27f6a..0d7b653e6 100644 --- a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -40,6 +40,10 @@ const ( // Progress states for status ProgressUpdating ProgressState = "Updating" ProgressReady ProgressState = "Ready" + + // Default port numbers + DefaultCqlPort = 9042 + DefaultBroadcastPort = 7000 ) // This type exists so there's no chance of pushing random strings to our progress status @@ -620,7 +624,7 @@ func (dc *CassandraDatacenter) GetNodePortCqlPort() int { } else if dc.Spec.Networking.NodePort.Cql != 0 { return dc.Spec.Networking.NodePort.Cql } else { - return 9042 + return DefaultCqlPort } } @@ -638,15 +642,15 @@ func (dc *CassandraDatacenter) GetNodePortBroadcastPort() int { } else if dc.Spec.Networking.NodePort.Broadcast != 0 { return dc.Spec.Networking.NodePort.Broadcast } else { - return 7000 + return DefaultBroadcastPort } } // GetContainerPorts will return the container ports for the pods in a statefulset based on the provided config func (dc *CassandraDatacenter) GetContainerPorts() ([]corev1.ContainerPort, error) { - cqlPort := 9042 - broadcastPort := 7000 + cqlPort := DefaultCqlPort + broadcastPort := DefaultBroadcastPort ports := []corev1.ContainerPort{ { diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types_test.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types_test.go index feb4314af..c6d171e72 100644 --- a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types_test.go +++ b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types_test.go @@ -236,13 +236,13 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) { want: []corev1.ContainerPort{ { Name: "native", - ContainerPort: 9042, + ContainerPort: DefaultCqlPort, }, { Name: "inter-node-msg", ContainerPort: 8609, }, { Name: "intra-node", - ContainerPort: 7000, + ContainerPort: DefaultBroadcastPort, }, { Name: "tls-intra-node", ContainerPort: 7001, @@ -263,13 +263,13 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) { want: []corev1.ContainerPort{ { Name: "native", - ContainerPort: 9042, + ContainerPort: DefaultCqlPort, }, { Name: "inter-node-msg", ContainerPort: 8609, }, { Name: "intra-node", - ContainerPort: 7000, + ContainerPort: DefaultBroadcastPort, }, { Name: "tls-intra-node", ContainerPort: 7001, @@ -292,13 +292,13 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) { want: []corev1.ContainerPort{ { Name: "native", - ContainerPort: 9042, + ContainerPort: DefaultCqlPort, }, { Name: "inter-node-msg", ContainerPort: 8609, }, { Name: "intra-node", - ContainerPort: 7000, + ContainerPort: DefaultBroadcastPort, }, { Name: "tls-intra-node", ContainerPort: 7001, diff --git a/operator/pkg/reconciliation/constructor.go b/operator/pkg/reconciliation/constructor.go index f4a037677..d34a91b54 100644 --- a/operator/pkg/reconciliation/constructor.go +++ b/operator/pkg/reconciliation/constructor.go @@ -42,7 +42,7 @@ func newServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Servi if !dc.IsNodePortEnabled() { service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{ - Name: "native", Port: 9042, TargetPort: intstr.FromInt(9042), + Name: "native", Port: api.DefaultCqlPort, TargetPort: intstr.FromInt(api.DefaultCqlPort), }, ) } From 380e7d9f2a9550c40be09d51a22b200563f1bb4f Mon Sep 17 00:00:00 2001 From: Ryan Springer Date: Tue, 28 Jul 2020 16:28:41 -0500 Subject: [PATCH 16/16] Do not remove native port from normal service when nodeport service is configured --- docs/user/README.md | 2 +- operator/pkg/reconciliation/constructor.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/user/README.md b/docs/user/README.md index 2d211304a..5da30eb83 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -379,7 +379,7 @@ The SSL versions of the ports may be requested: cqlSSL: 30010 broadcastSSL: 30020 -If any of the nodePort fields have been configured then a NodePort service will be created that routes from the specified external port to the identically numbered internal port. Cassandra will be configured to listen on the specified ports. Additionally, the standard service will no longer define a "native" port. +If any of the nodePort fields have been configured then a NodePort service will be created that routes from the specified external port to the identically numbered internal port. Cassandra will be configured to listen on the specified ports. # Using Your Cluster diff --git a/operator/pkg/reconciliation/constructor.go b/operator/pkg/reconciliation/constructor.go index d34a91b54..b80290932 100644 --- a/operator/pkg/reconciliation/constructor.go +++ b/operator/pkg/reconciliation/constructor.go @@ -32,21 +32,21 @@ func newServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Servi svcName := dc.GetDatacenterServiceName() service := makeGenericHeadlessService(dc) service.ObjectMeta.Name = svcName + + cqlPort := api.DefaultCqlPort + if dc.IsNodePortEnabled() { + cqlPort = dc.GetNodePortCqlPort() + } + service.Spec.Ports = []corev1.ServicePort{ + { + Name: "native", Port: int32(cqlPort), TargetPort: intstr.FromInt(cqlPort), + }, { Name: "mgmt-api", Port: 8080, TargetPort: intstr.FromInt(8080), }, } - // If NodePort is enabled, it will take control of the native port - if !dc.IsNodePortEnabled() { - service.Spec.Ports = append(service.Spec.Ports, - corev1.ServicePort{ - Name: "native", Port: api.DefaultCqlPort, TargetPort: intstr.FromInt(api.DefaultCqlPort), - }, - ) - } - addHashAnnotation(service) return service