[k8scluster/receiver] Add more e2e test coverage (open-telemetry#36114)

#### Description
- Add more e2e test coverage for cronjob, hpa, job, and statefulset
objects
- Updated the golden file (expected.yaml) used for test validation
- Added some test helpers to help create and delete multiple Kubernetes
test objects
- No .chloggen entry is needed for this PR because all the changes are
for testing enhancements

<!--Describe what testing was performed and which tests were added.-->
#### Testing
- Tested locally and in Github CI/CD

---------

Co-authored-by: Antoine Toulme <antoine@toulme.name>
Co-authored-by: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com>

internal/k8stest/k8s_objects.go

@@ -5,6 +5,8 @@ package k8stest // import "github.com/open-telemetry/opentelemetry-collector-con
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -53,3 +55,36 @@ func DeleteObject(client *K8sClient, obj *unstructured.Unstructured) error {
 		PropagationPolicy: &deletePolicy,
 	})
 }
+
+func CreateObjects(client *K8sClient, dir string) ([]*unstructured.Unstructured, error) {
+	var objs []*unstructured.Unstructured
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue // Skip directories
+		}
+		manifest, err := os.ReadFile(filepath.Join(dir, file.Name()))
+		if err != nil {
+			return nil, err
+		}
+		obj, err := CreateObject(client, manifest)
+		if err != nil {
+			return nil, err
+		}
+		objs = append(objs, obj)
+	}
+	return objs, nil
+}
+
+func DeleteObjects(client *K8sClient, objs []*unstructured.Unstructured) error {
+	for _, obj := range objs {
+		if err := DeleteObject(client, obj); err != nil {
+			return err
+		}
+	}
+	return nil
+}

receiver/k8sclusterreceiver/e2e_test.go

@@ -7,7 +7,6 @@ package k8sclusterreceiver
 
 import (
 	"context"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -26,7 +25,9 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest"
 )
 
+const expectedFile = "./testdata/e2e/expected.yaml"
 const testKubeConfig = "/tmp/kube-config-otelcol-e2e-testing"
+const testObjectsDir = "./testdata/e2e/testobjects/"
 
 // TestE2E tests the k8s cluster receiver with a real k8s cluster.
 // The test requires a prebuilt otelcontribcol image uploaded to a kind k8s cluster defined in
@@ -38,67 +39,110 @@ const testKubeConfig = "/tmp/kube-config-otelcol-e2e-testing"
 func TestE2E(t *testing.T) {
 
 	var expected pmetric.Metrics
-	expectedFile := filepath.Join("testdata", "e2e", "expected.yaml")
 	expected, err := golden.ReadMetrics(expectedFile)
 	require.NoError(t, err)
 
 	k8sClient, err := k8stest.NewK8sClient(testKubeConfig)
 	require.NoError(t, err)
 
+	// k8s test objs
+	testObjs, err := k8stest.CreateObjects(k8sClient, testObjectsDir)
+	require.NoErrorf(t, err, "failed to create objects")
+
+	t.Cleanup(func() {
+		require.NoErrorf(t, k8stest.DeleteObjects(k8sClient, testObjs), "failed to delete objects")
+	})
+
 	metricsConsumer := new(consumertest.MetricsSink)
 	shutdownSink := startUpSink(t, metricsConsumer)
 	defer shutdownSink()
 
 	testID := uuid.NewString()[:8]
 	collectorObjs := k8stest.CreateCollectorObjects(t, k8sClient, testID, "")
 
-	defer func() {
+	t.Cleanup(func() {
 		for _, obj := range append(collectorObjs) {
 			require.NoErrorf(t, k8stest.DeleteObject(k8sClient, obj), "failed to delete object %s", obj.GetName())
 		}
-	}()
+	})
 
 	wantEntries := 10 // Minimal number of metrics to wait for.
 	waitForData(t, wantEntries, metricsConsumer)
+	// golden.WriteMetrics(t, expectedFile, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1])
 
 	replaceWithStar := func(string) string { return "*" }
 	shortenNames := func(value string) string {
+		if strings.HasPrefix(value, "coredns") {
+			return "coredns"
+		}
+		if strings.HasPrefix(value, "kindnet") {
+			return "kindnet"
+		}
+		if strings.HasPrefix(value, "kube-apiserver") {
+			return "kube-apiserver"
+		}
 		if strings.HasPrefix(value, "kube-proxy") {
 			return "kube-proxy"
 		}
-		if strings.HasPrefix(value, "local-path-provisioner") {
-			return "local-path-provisioner"
+		if strings.HasPrefix(value, "kube-scheduler") {
+			return "kube-scheduler"
 		}
-		if strings.HasPrefix(value, "kindnet") {
-			return "kindnet"
+		if strings.HasPrefix(value, "kube-controller-manager") {
+			return "kube-controller-manager"
 		}
-		if strings.HasPrefix(value, "coredns") {
-			return "coredns"
+		if strings.HasPrefix(value, "local-path-provisioner") {
+			return "local-path-provisioner"
 		}
 		if strings.HasPrefix(value, "otelcol") {
 			return "otelcol"
 		}
+		if strings.HasPrefix(value, "test-k8scluster-receiver-cronjob") {
+			return "test-k8scluster-receiver-cronjob"
+		}
+		if strings.HasPrefix(value, "test-k8scluster-receiver-job") {
+			return "test-k8scluster-receiver-job"
+		}
 		return value
 	}
 	containerImageShorten := func(value string) string {
-		return value[(strings.LastIndex(value, "/") + 1):]
+		// Extracts the image name by removing the repository prefix.
+		// Also removes any architecture identifier suffix, if present, by applying shortenNames.
+		return shortenNames(value[(strings.LastIndex(value, "/") + 1):])
 	}
+
 	require.NoError(t, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1],
 		pmetrictest.IgnoreTimestamp(),
 		pmetrictest.IgnoreStartTimestamp(),
-		pmetrictest.IgnoreMetricValues("k8s.deployment.desired", "k8s.deployment.available", "k8s.container.restarts", "k8s.container.cpu_request", "k8s.container.memory_request", "k8s.container.memory_limit"),
+		pmetrictest.IgnoreMetricValues(
+			"k8s.container.cpu_request",
+			"k8s.container.memory_limit",
+			"k8s.container.memory_request",
+			"k8s.container.restarts",
+			"k8s.cronjob.active_jobs",
+			"k8s.deployment.available",
+			"k8s.deployment.desired",
+			"k8s.job.active_pods",
+			"k8s.job.desired_successful_pods",
+			"k8s.job.failed_pods",
+			"k8s.job.max_parallel_pods",
+			"k8s.job.successful_pods"),
+		pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten),
+		pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar),
 		pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames),
-		pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames),
-		pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames),
 		pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames),
+		pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames),
 		pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar),
+		pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames),
 		pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar),
-		pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar),
-		pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar),
-		pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar),
-		pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar),
-		pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar),
-		pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten),
+		pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar),
 		pmetrictest.IgnoreScopeVersion(),
 		pmetrictest.IgnoreResourceMetricsOrder(),
 		pmetrictest.IgnoreMetricsOrder(),