Turn ingress TestCheck into an E2e test

cmd/ingress/ingress.go

@@ -3,6 +3,7 @@ package ingress
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"time"
 
@@ -28,9 +29,15 @@ type Ingress struct {
 	Namespace        string
 	Clientset        *kubernetes.Clientset
 	NoDnsCheck       bool
+	NoHTTPCheck      bool
 	IngressClassName string
 	ResourceName     string
 	ExternalHostname string
+	// HTTPCheckEndpoint is the HTTP endpoint that is requested to check the service.
+	// If not set, it defaults to http://<ExternalHostname>/
+	// This is usually set to the LoadBalancer IP of the Ingress Controller Service,
+	// in case the external hostname is not resolvable.
+	HTTPCheckEndpoint string
 }
 
 func NewIngress(checker *cmd.Checker, noDnsCheck bool) (*Ingress, error) {
@@ -98,6 +105,16 @@ func (i *Ingress) Check() error {
 		}
 	}
 
+	if i.NoHTTPCheck {
+		i.Chatwork.AddMessage("Skip HTTP Check\n")
+		i.Logger().Info("Skip HTTP Check")
+	} else {
+		if err := i.checkHTTP(); err != nil {
+			return err
+		}
+
+	}
+
 	i.Chatwork.AddMessage("Ingress check finished\n")
 	return nil
 }
@@ -187,7 +204,7 @@ func (i *Ingress) createDeploymentObject() *appsv1.Deployment {
 								{
 									Name:          "http",
 									Protocol:      apiv1.ProtocolTCP,
-									ContainerPort: 8080,
+									ContainerPort: 80,
 								},
 							},
 						},
@@ -213,7 +230,7 @@ func (i *Ingress) createServiceObject() *apiv1.Service {
 				{
 					Protocol:   apiv1.ProtocolTCP,
 					Port:       80,
-					TargetPort: intstr.FromInt(8080),
+					TargetPort: intstr.FromInt(80),
 				},
 			},
 		},
@@ -308,3 +325,46 @@ func (i *Ingress) checkDNSRecord() error {
 
 	return nil
 }
+
+// Tries to access the service endpoint via HTTP
+// and see if it returns 200 OK.
+func (i *Ingress) checkHTTP() error {
+	var endpoint string
+	if i.HTTPCheckEndpoint != "" {
+		endpoint = i.HTTPCheckEndpoint
+	} else {
+		endpoint = fmt.Sprintf("http://%s/", i.ExternalHostname)
+	}
+
+	i.Logger().Infof("Check HTTP for: %s", endpoint)
+	err := wait.PollUntilContextTimeout(i.Ctx, 10*time.Second, i.Timeout, false, func(ctx context.Context) (bool, error) {
+		req, err := http.NewRequest("GET", endpoint, nil)
+		if err != nil {
+			return false, err
+		}
+		req.Host = i.ExternalHostname
+
+		i.Logger().Infof("Requesting %s with headers %v", endpoint, req.Header)
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			i.Logger().Warn(err)
+			return false, nil
+		}
+
+		if resp.StatusCode != 200 {
+			i.Logger().Infof("HTTP Status Code is not 200: %d", resp.StatusCode)
+			return false, nil
+		}
+
+		i.Logger().Info("HTTP Status Code is 200")
+		i.Chatwork.AddMessage("HTTP Status Code is 200\n")
+		return true, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("waiting for HTTP service to be ready: %w", err)
+	}
+
+	return nil
+}

cmd/ingress/ingress_test.go

@@ -4,14 +4,16 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/chatwork/kibertas/cmd"
 	"github.com/chatwork/kibertas/config"
-	"github.com/chatwork/kibertas/util"
 	"github.com/chatwork/kibertas/util/notify"
 	"github.com/mumoshu/testkit"
+	"github.com/stretchr/testify/require"
 
 	"github.com/sirupsen/logrus"
 )
@@ -34,6 +36,10 @@ func TestNewIngress(t *testing.T) {
 }
 
 func TestCheck(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test in short mode.")
+	}
+
 	t.Parallel()
 	logger := func() *logrus.Entry {
 		return logrus.NewEntry(logrus.New())
@@ -53,18 +59,43 @@ func TestCheck(t *testing.T) {
 
 	kc := h.KubernetesCluster(t)
 
+	// Start cloud-provider-kind to manage service type=LoadBalancer
+	//
+	// This requiers cloud-provider-kind to be installed in the PATH.
+	// Follow https://github.com/kubernetes-sigs/cloud-provider-kind?tab=readme-ov-file#install to install it.
+	bin, err := exec.LookPath("cloud-provider-kind")
+	if bin == "" {
+		t.Fatalf("cloud-provider-kind not found in PATH: %s", os.Getenv("PATH"))
+	}
+	require.NoError(t, err)
+
+	handle := StartProcess(t, bin)
+	defer handle.Stop(t)
+
 	helm := testkit.NewHelm(kc.KubeconfigPath)
 	// See https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler#tldr
 	helm.AddRepo(t, "ingress-nginx", "https://kubernetes.github.io/ingress-nginx")
 
 	ingressNginxNs := "default"
 	helm.UpgradeOrInstall(t, "my-ingress-nginx", "ingress-nginx/ingress-nginx", func(hc *testkit.HelmConfig) {
-		hc.Values = map[string]interface{}{}
+		hc.Values = map[string]interface{}{
+			"rbac": map[string]interface{}{
+				"create": true,
+			},
+		}
 
 		hc.Namespace = ingressNginxNs
 	})
 
-	namespace := fmt.Sprintf("ingress-test-%d%02d%02d-%s", now.Year(), now.Month(), now.Day(), util.GenerateRandomString(5))
+	kctl := testkit.NewKubectl(kc.KubeconfigPath)
+
+	// Get the external IP of the ingress-nginx service
+	ingressNginxSvcLBIP := kctl.Capture(t, "get", "svc", "-n", ingressNginxNs, "my-ingress-nginx-controller", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}")
+	t.Logf("ingress-nginx service LB IP: %s", ingressNginxSvcLBIP)
+
+	// We intentionally make the test namespace deterministic to avoid ingress path
+	// conflicts among test namespaces across test runs
+	namespace := fmt.Sprintf("ingress-test-%d%02d%02d", now.Year(), now.Month(), now.Day())
 
 	os.Setenv("KUBECONFIG", kc.KubeconfigPath)
 
@@ -76,13 +107,14 @@ func TestCheck(t *testing.T) {
 	// kindとingress-nginxがある前提
 	// レコードは作れないのでNoDnsCheckをtrueにする
 	ingress := &Ingress{
-		Checker:          cmd.NewChecker(context.Background(), true, logger, chatwork, "test", 1*time.Minute),
-		Namespace:        namespace,
-		Clientset:        k8sclient,
-		NoDnsCheck:       true,
-		IngressClassName: "nginx",
-		ResourceName:     "sample",
-		ExternalHostname: "sample.example.com",
+		Checker:           cmd.NewChecker(context.Background(), true, logger, chatwork, "test", 1*time.Minute),
+		Namespace:         namespace,
+		Clientset:         k8sclient,
+		NoDnsCheck:        true,
+		IngressClassName:  "nginx",
+		ResourceName:      "sample",
+		ExternalHostname:  "sample.example.com",
+		HTTPCheckEndpoint: "http://" + ingressNginxSvcLBIP + "/",
 	}
 
 	err = ingress.Check()
@@ -91,6 +123,41 @@ func TestCheck(t *testing.T) {
 	}
 }
 
+type ProcessHandle struct {
+	proc *os.Process
+}
+
+// Sends a SIGTERM to the process
+func (h *ProcessHandle) Stop(t *testing.T) {
+	t.Helper()
+
+	if err := h.proc.Signal(syscall.SIGTERM); err != nil {
+		t.Errorf("Failed to send SIGTERM to the process: %s", err)
+	}
+
+	if _, err := h.proc.Wait(); err != nil {
+		t.Errorf("Failed to wait for the process to exit: %s", err)
+	}
+}
+
+func StartProcess(t *testing.T, name string) *ProcessHandle {
+	t.Helper()
+
+	handle := &ProcessHandle{}
+
+	proc, err := os.StartProcess(name, []string{}, &os.ProcAttr{
+		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to start process: %s", err)
+	}
+
+	handle.proc = proc
+
+	return handle
+}
+
 func TestCheckDNSRecord(t *testing.T) {
 	logger := func() *logrus.Entry {
 		return logrus.NewEntry(logrus.New())