feat: detect node port range from bundle

cmd/serve.go

@@ -24,6 +24,7 @@ type serveOptions struct {
 	proxyAddress          string
 	envtestArch           string
 	serviceClusterIPRange string
+	serviceNodePortRange  string
 }
 
 // NewServeCommand serves the provided bundle.
@@ -64,6 +65,11 @@ func NewServeCommand(out output.Output) *cobra.Command {
 		"override k8s api server service ClusterIP range. Mask must be >= /12 range.",
 	)
 
+	cmd.Flags().StringVar(
+		&options.serviceNodePortRange, "service-node-port-range", options.serviceNodePortRange,
+		"override k8s api server service node port range",
+	)
+
 	return cmd
 }
 
@@ -134,6 +140,15 @@ func startK8sServer(
 		testEnv.ControlPlane.GetAPIServer().Configure().Append("service-cluster-ip-range", serviceClusterIPRange)
 	}
 
+	serviceNodePortRange, err := resolveServiceNodePortRange(
+		opts.serviceNodePortRange, supportBundle, out)
+	if err != nil {
+		return nil, err
+	}
+	if serviceNodePortRange != "" {
+		testEnv.ControlPlane.GetAPIServer().Configure().Append("service-node-port-range", serviceNodePortRange)
+	}
+
 	_, err = testEnv.Start()
 	if err != nil {
 		return nil, err
@@ -142,6 +157,36 @@ func startK8sServer(
 	return testEnv, nil
 }
 
+func resolveServiceNodePortRange(
+	nodePortRangeFromFlag string,
+	supportBundle bundle.Bundle,
+	out output.Output,
+) (string, error) {
+	// Manually provided via CLI flag
+	if nodePortRangeFromFlag != "" {
+		return nodePortRangeFromFlag, nil
+	}
+
+	// Detected from the bundle
+	nodePortRangeFromBundle, err := bundle.DetectServiceNodePortRange(supportBundle)
+	if err != nil {
+		return "", err
+	}
+	if nodePortRangeFromBundle != "" {
+		out.V(1).Infof("Detected service node port range: %s", nodePortRangeFromBundle)
+		return nodePortRangeFromBundle, nil
+	}
+
+	// Fallback default
+	out.Warnf(
+		"Service node port range could not be detected from support bundle, using default %q. Use "+
+			"%q flag to override this value.",
+		kubernetes.DefaultServiceNodePortRange,
+		"service-node-port-range",
+	)
+	return kubernetes.DefaultServiceNodePortRange, nil
+}
+
 func resolveServiceClusterIPRange(
 	ipRangeFromFlag string,
 	supportBundle bundle.Bundle,

pkg/bundle/service_ip_range.go

@@ -9,39 +9,87 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+const apiServerContainerName = "kube-apiserver"
+
 // DetectServiceSubnetRange attempts to determine service ip range value provided
 // to k8s api server, so that local version can be launched with same argument.
 // So far the function tries to parse value from `kube-apiserver` pod.
 // Other potential locations for parsing this value:
 // - CAPI cluster resource
 // - KIND kubeadm config.
 func DetectServiceSubnetRange(b Bundle) (string, error) {
+	apiServerPod, err := findKubeApiserverPod(b)
+	if err != nil {
+		return "", err
+	}
+
+	// Some bundles collected from managed providers, like gke, eks would not have
+	// the kube-apiserver pod.
+	if apiServerPod == nil {
+		return "", nil
+	}
+
+	return parseIPRangeArg(apiServerPod)
+}
+
+// DetectServiceNodePortRange attempts to determine service node port range value provided
+// to k8s api server, so that local version can be launched with same argument.
+func DetectServiceNodePortRange(b Bundle) (string, error) {
+	apiServerPod, err := findKubeApiserverPod(b)
+	if err != nil {
+		return "", err
+	}
+
+	// Some bundles collected from managed providers, like gke, eks would not have
+	// the kube-apiserver pod.
+	if apiServerPod == nil {
+		return "", nil
+	}
+
+	return parseNodePortRangeArg(apiServerPod)
+}
+
+func findKubeApiserverPod(b Bundle) (*corev1.Pod, error) {
 	path := filepath.Join(b.Layout().ClusterResources(), "pods", "kube-system.json")
 	list, err := LoadResourcesFromFile(b, path)
 	if err != nil {
-		return "", fmt.Errorf("failed to load pods from file %q: %w", path, err)
+		return nil, fmt.Errorf("failed to load pods from file %q: %w", path, err)
 	}
 
 	for i := range list.Items {
 		pod := &corev1.Pod{}
 		if err := runtime.DefaultUnstructuredConverter.
 			FromUnstructured(list.Items[i].UnstructuredContent(), &pod); err != nil {
-			return "", err
+			return nil, err
 		}
 
-		if !isKubeApiserverPod(pod) {
+		if isKubeApiserverPod(pod) {
+			return pod, nil
+		}
+	}
+
+	return nil, nil
+}
+
+func parseNodePortRangeArg(pod *corev1.Pod) (string, error) {
+	for _, c := range pod.Spec.Containers {
+		if c.Name != apiServerContainerName {
 			continue
 		}
 
-		return parseIPRangeArg(pod)
+		for _, arg := range c.Command {
+			if strings.HasPrefix(arg, "--service-node-port-range=") {
+				return strings.TrimPrefix(arg, "--service-node-port-range="), nil
+			}
+		}
 	}
 
 	return "", nil
 }
 
 func parseIPRangeArg(pod *corev1.Pod) (string, error) {
 	for _, c := range pod.Spec.Containers {
-		if c.Name != "kube-apiserver" {
+		if c.Name != apiServerContainerName {
 			continue
 		}
 

pkg/kubernetes/constants.go

@@ -0,0 +1,17 @@
+package kubernetes
+
+const (
+	// DefaultServiceClusterIPRange is the fallback value for service ClusterIP
+	// range of k8s API server configuration. This value is used when the cluster
+	// cannot be detected from the bundle itself. This happens mostly for managed
+	// k8s platforms like EKS, AKS which will not return API server pod in the list
+	// of pods.
+	DefaultServiceClusterIPRange = "10.0.0.0/12"
+
+	// DefaultServiceNodePortRange is the fallback value for service node port
+	// range of k8s API server configuration. This value is used when the cluster
+	// cannot be detected from the bundle itself. This happens mostly for managed
+	// k8s platforms like EKS, AKS which will not return API server pod in the list
+	// of pods.
+	DefaultServiceNodePortRange = "30000-32767"
+)

pkg/kubernetes/kubeconfig.go

@@ -10,13 +10,6 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
 
-// DefaultServiceClusterIPRange is the fallback value for service ClusterIP
-// range of k8s API server configuration. This value is used when the cluster
-// cannot be detected from the bundle itself. This happens mostly for managed
-// k8s platforms like EKS, AKS which will not return API server pod in the list
-// of pods.
-const DefaultServiceClusterIPRange = "10.0.0.0/12"
-
 // WriteProxyKubeconfig creates a KUBECONFIG file for http proxy server. If path
 // for kubeconfig is not provided then default value is create in `CWD`.
 func WriteProxyKubeconfig(host, path string) (string, error) {