fix: clean up imagejob when eraser-manager pod is deleted (#875)

Makefile

@@ -156,6 +156,13 @@ busybox-img:
 		--build-arg IMG=$(BUSYBOX_BASE_IMG) test/e2e/test-data
 BUSYBOX_IMG=busybox-e2e-test:latest
 
+collector-dummy-img:
+	docker build -t $(COLLECTOR_REPO):dummy \
+		-f test/e2e/test-data/Dockerfile.dummyCollector \
+		test/e2e/test-data
+COLLECTOR_IMAGE_DUMMY=$(COLLECTOR_REPO):dummy
+
+
 vulnerable-img:
 	docker pull $(VULNERABLE_IMG)
 
@@ -171,7 +178,7 @@ non-vulnerable-img:
 		-t ${NON_VULNERABLE_IMG} \
 		--target non-vulnerable .
 
-e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img
+e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img collector-dummy-img
 	for test in $(E2E_TESTS); do \
 		CGO_ENABLED=0 \
             PROJECT_ABSOLUTE_PATH=$(CURDIR) \
@@ -185,6 +192,7 @@ e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img
 			COLLECTOR_IMAGE=${COLLECTOR_IMG} \
 			SCANNER_IMAGE=${TRIVY_SCANNER_IMG} \
 			BUSYBOX_IMAGE=${BUSYBOX_IMG} \
+			COLLECTOR_IMAGE_DUMMY=${COLLECTOR_IMAGE_DUMMY} \
 			VULNERABLE_IMAGE=${VULNERABLE_IMG} \
 			NON_VULNERABLE_IMAGE=${NON_VULNERABLE_IMG} \
 			EOL_IMAGE=${EOL_IMG} \

controllers/imagejob/imagejob_controller.go

@@ -255,7 +255,11 @@ func (r *Reconciler) handleRunningJob(ctx context.Context, imageJob *eraserv1.Im
 		Namespace: namespace,
 	}, &template)
 	if err != nil {
-		return err
+		imageJob.Status = eraserv1.ImageJobStatus{
+			Phase:       eraserv1.PhaseFailed,
+			DeleteAfter: controllerUtils.After(time.Now(), 1),
+		}
+		return r.updateJobStatus(ctx, imageJob)
 	}
 
 	listOpts := podListOptions(&template)

test/e2e/test-data/Dockerfile.dummyCollector

@@ -0,0 +1,3 @@
+FROM busybox:latest
+
+ENTRYPOINT ["yes"]

test/e2e/tests/collector_delete_manager/eraser_test.go

@@ -0,0 +1,90 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/eraser-dev/eraser/test/e2e/util"
+
+	eraserv1alpha1 "github.com/eraser-dev/eraser/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestDeleteManager(t *testing.T) {
+	deleteManagerFeat := features.New("Deleting manager pod while current ImageJob is running should delete ImageJob and restart").
+		Assess("Wait for eraser pods running", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			c, err := cfg.NewClient()
+			if err != nil {
+				t.Fatal("Failed to create new client", err)
+			}
+
+			err = wait.For(
+				util.NumPodsPresentForLabel(ctx, c, 3, util.ImageJobTypeLabelKey+"="+util.CollectorLabel),
+				wait.WithTimeout(time.Minute*2),
+				wait.WithInterval(time.Millisecond*500),
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			return ctx
+		}).
+		Assess("Delete controller-manager pod", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			c, err := cfg.NewClient()
+			if err != nil {
+				t.Fatal("Failed to create new client", err)
+			}
+
+			// get manager pod
+			var podList corev1.PodList
+			err = c.Resources().List(ctx, &podList, func(o *metav1.ListOptions) {
+				o.LabelSelector = labels.SelectorFromSet(map[string]string{util.ManagerLabelKey: util.ManagerLabelValue}).String()
+			})
+			if err != nil {
+				t.Errorf("could not list manager pods: %v", err)
+			}
+
+			if len(podList.Items) != 1 {
+				t.Error("incorrect number of manager pods: ", len(podList.Items))
+			}
+
+			// get current ImageJob before deleting manager pod
+			var jobList eraserv1alpha1.ImageJobList
+			err = c.Resources().List(ctx, &jobList)
+			if err != nil {
+				t.Errorf("could not list ImageJob: %v", err)
+			}
+
+			t.Log("job", jobList.Items[0], "name", jobList.Items[0].Name)
+
+			if len(jobList.Items) != 1 {
+				t.Error("incorrect number of ImageJobs: ", len(jobList.Items))
+			}
+
+			// delete manager pod
+			if err := util.KubectlDelete(cfg.KubeconfigFile(), util.TestNamespace, []string{"pod", podList.Items[0].Name}); err != nil {
+				t.Error("unable to delete eraser-controller-manager pod")
+			}
+
+			// wait for deletion of ImageJob
+			err = wait.For(conditions.New(c.Resources()).ResourcesDeleted(&jobList), wait.WithTimeout(util.Timeout))
+			if err != nil {
+				t.Errorf("error waiting for ImageJob to be deleted: %v", err)
+			}
+
+			return ctx
+		}).
+		Feature()
+
+	util.Testenv.Test(t, deleteManagerFeat)
+}

test/e2e/tests/collector_delete_manager/main_test.go

@@ -0,0 +1,59 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"os"
+	"testing"
+
+	eraserv1alpha1 "github.com/eraser-dev/eraser/api/v1alpha1"
+	"github.com/eraser-dev/eraser/test/e2e/util"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+)
+
+func TestMain(m *testing.M) {
+	utilruntime.Must(eraserv1alpha1.AddToScheme(scheme.Scheme))
+
+	removerImage := util.ParsedImages.RemoverImage
+	managerImage := util.ParsedImages.ManagerImage
+	collectorImage := util.ParsedImages.CollectorImage
+
+	util.Testenv = env.NewWithConfig(envconf.New())
+	// Create KinD Cluster
+	util.Testenv.Setup(
+		envfuncs.CreateKindClusterWithConfig(util.KindClusterName, util.NodeVersion, util.KindConfigPath),
+		envfuncs.CreateNamespace(util.TestNamespace),
+		util.LoadImageToCluster(util.KindClusterName, util.ManagerImage, util.ManagerTarballPath),
+		util.LoadImageToCluster(util.KindClusterName, util.RemoverImage, util.RemoverTarballPath),
+		util.LoadImageToCluster(util.KindClusterName, util.RemoverImage, util.RemoverTarballPath),
+		util.LoadImageToCluster(util.KindClusterName, util.CollectorDummyImage, ""),
+		util.LoadImageToCluster(util.KindClusterName, util.NonVulnerableImage, ""),
+		util.HelmDeployLatestEraserRelease(util.TestNamespace,
+			"--set", util.ScannerEnable.Set("false"),
+			"--set", util.CollectorEnable.Set("false"),
+			"--set", util.RemoverImageRepo.Set(removerImage.Repo),
+			"--set", util.RemoverImageTag.Set(removerImage.Tag),
+			"--set", util.ManagerImageRepo.Set(managerImage.Repo),
+			"--set", util.ManagerImageTag.Set(managerImage.Tag),
+		),
+		util.UpgradeEraserHelm(util.TestNamespace,
+			"--set", util.ScannerEnable.Set("false"),
+			"--set", util.RemoverImageRepo.Set(removerImage.Repo),
+			"--set", util.RemoverImageTag.Set(removerImage.Tag),
+			"--set", util.CollectorEnable.Set("true"),
+			"--set", util.CollectorImageRepo.Set(collectorImage.Repo),
+			"--set", util.CollectorImageTag.Set("dummy"),
+			"--set", util.ManagerImageRepo.Set(managerImage.Repo),
+			"--set", util.ManagerImageTag.Set(managerImage.Tag),
+			"--set", util.CleanupOnSuccessDelay.Set("2m"),
+		),
+	).Finish(
+		envfuncs.DestroyKindCluster(util.KindClusterName),
+	)
+	os.Exit(util.Testenv.Run(m))
+}

test/e2e/util/utils.go

@@ -82,18 +82,21 @@ const (
 	CollectorLabel       = "collector"
 	ManualLabel          = "manual"
 	ImageJobTypeLabelKey = "eraser.sh/type"
+	ManagerLabelKey      = "control-plane"
+	ManagerLabelValue    = "controller-manager"
 )
 
 var (
-	Testenv            env.Environment
-	RemoverImage       = os.Getenv("REMOVER_IMAGE")
-	ManagerImage       = os.Getenv("MANAGER_IMAGE")
-	CollectorImage     = os.Getenv("COLLECTOR_IMAGE")
-	ScannerImage       = os.Getenv("SCANNER_IMAGE")
-	VulnerableImage    = os.Getenv("VULNERABLE_IMAGE")
-	NonVulnerableImage = os.Getenv("NON_VULNERABLE_IMAGE")
-	EOLImage           = os.Getenv("EOL_IMAGE")
-	BusyboxImage       = os.Getenv("BUSYBOX_IMAGE")
+	Testenv             env.Environment
+	RemoverImage        = os.Getenv("REMOVER_IMAGE")
+	ManagerImage        = os.Getenv("MANAGER_IMAGE")
+	CollectorImage      = os.Getenv("COLLECTOR_IMAGE")
+	ScannerImage        = os.Getenv("SCANNER_IMAGE")
+	VulnerableImage     = os.Getenv("VULNERABLE_IMAGE")
+	NonVulnerableImage  = os.Getenv("NON_VULNERABLE_IMAGE")
+	EOLImage            = os.Getenv("EOL_IMAGE")
+	BusyboxImage        = os.Getenv("BUSYBOX_IMAGE")
+	CollectorDummyImage = os.Getenv("COLLECTOR_IMAGE_DUMMY")
 
 	RemoverTarballPath   = os.Getenv("REMOVER_TARBALL_PATH")
 	ManagerTarballPath   = os.Getenv("MANAGER_TARBALL_PATH")