Make CL2 accept multiple master ips.

This change should be no-op, it only changes the CL2 api (flags/envs) to accept multiple values as master ip. The api change should be backward compatible, i.e. if flag/env was set to a single master-ip it will continue to work. Actual changes supporting multi-master clusters will come in separate PRs.
kubernetes · Jul 3, 2019 · ed885ba · ed885ba
1 parent 73976e9
commit ed885ba
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 31 deletions.
diff --git a/clusterloader2/cmd/clusterloader.go b/clusterloader2/cmd/clusterloader.go
@@ -57,8 +57,9 @@ func initClusterFlags() {
 	flags.IntEnvVar(&clusterLoaderConfig.ClusterConfig.Nodes, "nodes", "NUM_NODES", 0, "number of nodes")
 	flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.Provider, "provider", "PROVIDER", "", "Cluster provider")
 	flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterName, "mastername", "MASTER_NAME", "", "Name of the masternode")
-	flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterIP, "masterip", "MASTER_IP", "", "Hostname/IP of the masternode")
-	flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterInternalIP, "master-internal-ip", "MASTER_INTERNAL_IP", "", "Cluster internal/private IP of the master vm")
+	// TODO(#595): Change the name of the MASTER_IP and MASTER_INTERNAL_IP flags and vars to plural
+	flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterIPs, "masterip", "MASTER_IP", nil /*defaultValue*/, "Hostname/IP of the master node, supports multiple values when separated by commas")
+	flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterInternalIPs, "master-internal-ip", "MASTER_INTERNAL_IP", nil /*defaultValue*/, "Cluster internal/private IP of the master vm, supports multiple values when separated by commas")
 	flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.KubemarkRootKubeConfigPath, "kubemark-root-kubeconfig", "KUBEMARK_ROOT_KUBECONFIG", "",
 		"Path the to kubemark root kubeconfig file, i.e. kubeconfig of the cluster where kubemark cluster is run. Ignored if provider != kubemark")
 }
@@ -112,20 +113,20 @@ func completeConfig(m *framework.MultiClientSet) error {
 			klog.Errorf("Getting master name error: %v", err)
 		}
 	}
-	if clusterLoaderConfig.ClusterConfig.MasterIP == "" {
-		masterIP, err := util.GetMasterIP(m.GetClient(), corev1.NodeExternalIP)
+	if len(clusterLoaderConfig.ClusterConfig.MasterIPs) == 0 {
+		masterIPs, err := util.GetMasterIPs(m.GetClient(), corev1.NodeExternalIP)
 		if err == nil {
-			clusterLoaderConfig.ClusterConfig.MasterIP = masterIP
-			klog.Infof("ClusterConfig.MasterIP set to %v", masterIP)
+			clusterLoaderConfig.ClusterConfig.MasterIPs = masterIPs
+			klog.Infof("ClusterConfig.MasterIP set to %v", masterIPs)
 		} else {
 			klog.Errorf("Getting master external ip error: %v", err)
 		}
 	}
-	if clusterLoaderConfig.ClusterConfig.MasterInternalIP == "" {
-		masterIP, err := util.GetMasterIP(m.GetClient(), corev1.NodeInternalIP)
+	if len(clusterLoaderConfig.ClusterConfig.MasterInternalIPs) == 0 {
+		masterIPs, err := util.GetMasterIPs(m.GetClient(), corev1.NodeInternalIP)
 		if err == nil {
-			clusterLoaderConfig.ClusterConfig.MasterInternalIP = masterIP
-			klog.Infof("ClusterConfig.MasterInternalIP set to %v", masterIP)
+			clusterLoaderConfig.ClusterConfig.MasterInternalIPs = masterIPs
+			klog.Infof("ClusterConfig.MasterInternalIP set to %v", masterIPs)
 		} else {
 			klog.Errorf("Getting master internal ip error: %v", err)
 		}

diff --git a/clusterloader2/pkg/config/cluster.go b/clusterloader2/pkg/config/cluster.go
@@ -28,12 +28,29 @@ type ClusterLoaderConfig struct {
 
 // ClusterConfig is a structure that represents cluster description.
 type ClusterConfig struct {
-	KubeConfigPath string `json: kubeConfigPath`
-	Nodes          int    `json: nodes`
-	Provider       string `json: provider`
-	// TODO(krzysied): Add support for HA cluster with more than one master.
-	MasterIP                   string `json: masterIP`
-	MasterInternalIP           string `json: masterInternalIP`
-	MasterName                 string `json: masterName`
-	KubemarkRootKubeConfigPath string `json: kubemarkRootKubeConfigPath`
+	KubeConfigPath             string   `json: kubeConfigPath`
+	Nodes                      int      `json: nodes`
+	Provider                   string   `json: provider`
+	MasterIPs                  []string `json: masterIPs`
+	MasterInternalIPs          []string `json: masterInternalIPs`
+	MasterName                 string   `json: masterName`
+	KubemarkRootKubeConfigPath string   `json: kubemarkRootKubeConfigPath`
+}
+
+// GetMasterIp returns the first master ip, added for backward compatibility.
+// TODO(mmatt): Remove this method once all the codebase is migrated to support multiple masters.
+func (c *ClusterConfig) GetMasterIp() string {
+	if len(c.MasterIPs) > 0 {
+		return c.MasterIPs[0]
+	}
+	return ""
+}
+
+// GetMasterInternalIp returns the first internal master ip, added for backward compatibility.
+// TODO(mmatt): Remove this method once all the codebase is migrated to support multiple masters.
+func (c *ClusterConfig) GetMasterInternalIp() string {
+	if len(c.MasterInternalIPs) > 0 {
+		return c.MasterInternalIPs[0]
+	}
+	return ""
 }
diff --git a/clusterloader2/pkg/measurement/common/simple/etcd_metrics.go b/clusterloader2/pkg/measurement/common/simple/etcd_metrics.go
@@ -66,7 +66,7 @@ func (e *etcdMetricsMeasurement) Execute(config *measurement.MeasurementConfig)
 	if err != nil {
 		return nil, err
 	}
-	host, err := util.GetStringOrDefault(config.Params, "host", config.ClusterFramework.GetClusterConfig().MasterIP)
+	host, err := util.GetStringOrDefault(config.Params, "host", config.ClusterFramework.GetClusterConfig().GetMasterIp())
 	if err != nil {
 		return nil, err
 	}

diff --git a/clusterloader2/pkg/measurement/common/simple/profile.go b/clusterloader2/pkg/measurement/common/simple/profile.go
@@ -59,7 +59,7 @@ func createProfileConfig(config *measurement.MeasurementConfig) (*profileConfig,
 	if pc.provider, err = util.GetStringOrDefault(config.Params, "provider", config.ClusterFramework.GetClusterConfig().Provider); err != nil {
 		return nil, err
 	}
-	if pc.host, err = util.GetStringOrDefault(config.Params, "host", config.ClusterFramework.GetClusterConfig().MasterIP); err != nil {
+	if pc.host, err = util.GetStringOrDefault(config.Params, "host", config.ClusterFramework.GetClusterConfig().GetMasterIp()); err != nil {
 		return nil, err
 	}
 	return pc, nil

diff --git a/clusterloader2/pkg/measurement/common/simple/resource_usage.go b/clusterloader2/pkg/measurement/common/simple/resource_usage.go
@@ -66,7 +66,7 @@ func (e *resourceUsageMetricMeasurement) Execute(config *measurement.Measurement
 		if err != nil {
 			return nil, err
 		}
-		host, err := util.GetStringOrDefault(config.Params, "host", config.ClusterFramework.GetClusterConfig().MasterIP)
+		host, err := util.GetStringOrDefault(config.Params, "host", config.ClusterFramework.GetClusterConfig().GetMasterIp())
 		if err != nil {
 			return nil, err
 		}

diff --git a/clusterloader2/pkg/measurement/common/simple/scheduler_latency.go b/clusterloader2/pkg/measurement/common/simple/scheduler_latency.go
@@ -65,7 +65,7 @@ func (s *schedulerLatencyMeasurement) Execute(config *measurement.MeasurementCon
 	if err != nil {
 		return nil, err
 	}
-	masterIP, err := util.GetStringOrDefault(config.Params, "masterIP", config.ClusterFramework.GetClusterConfig().MasterIP)
+	masterIP, err := util.GetStringOrDefault(config.Params, "masterIP", config.ClusterFramework.GetClusterConfig().GetMasterIp())
 	if err != nil {
 		return nil, err
 	}

diff --git a/clusterloader2/pkg/prometheus/prometheus.go b/clusterloader2/pkg/prometheus/prometheus.go
@@ -228,9 +228,9 @@ func dumpAdditionalLogsOnPrometheusSetupFailure(k8sClient kubernetes.Interface)
 }
 
 func getMasterIp(clusterConfig config.ClusterConfig) (string, error) {
-	if clusterConfig.MasterInternalIP != "" {
-		klog.Infof("Using internal master ip (%s) to monitor master's components", clusterConfig.MasterInternalIP)
-		return clusterConfig.MasterInternalIP, nil
+	if clusterConfig.GetMasterInternalIp() != "" {
+		klog.Infof("Using internal master ip (%s) to monitor master's components", clusterConfig.GetMasterInternalIp())
+		return clusterConfig.GetMasterInternalIp(), nil
 	}
 	return "", fmt.Errorf("internal master ip not available")
 }
diff --git a/clusterloader2/pkg/util/cluster.go b/clusterloader2/pkg/util/cluster.go
@@ -135,21 +135,25 @@ func GetMasterName(c clientset.Interface) (string, error) {
 	return "", fmt.Errorf("master node not found")
 }
 
-// GetMasterIP returns master node ip of the given type.
-func GetMasterIP(c clientset.Interface, addressType corev1.NodeAddressType) (string, error) {
+// GetMasterIPs returns master node ips of the given type.
+func GetMasterIPs(c clientset.Interface, addressType corev1.NodeAddressType) ([]string, error) {
 	nodeList, err := client.ListNodes(c)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
+	var ips []string
 	for i := range nodeList {
 		if system.IsMasterNode(nodeList[i].Name) {
 			for _, address := range nodeList[i].Status.Addresses {
 				if address.Type == addressType && address.Address != "" {
-					return address.Address, nil
+					ips = append(ips, address.Address)
+					break
 				}
 			}
-			return "", fmt.Errorf("%s IP of the master not found", addressType)
 		}
 	}
-	return "", fmt.Errorf("master node not found")
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("didn't find any %s master IPs", addressType)
+	}
+	return ips, nil
 }