e2e: Fix fluentd test for AL2023, and for CI using OIDC

cmd/fluent/fluent_test.go

@@ -6,6 +6,8 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -22,19 +24,45 @@ func TestFluentE2E(t *testing.T) {
 		t.Skip("Skipping test in short mode.")
 	}
 
+	prefix := os.Getenv("PREFIX")
+	if prefix == "" {
+		t.Skip("PREFIX is not set")
+	}
+
 	vpcID := os.Getenv("VPC_ID")
 	if vpcID == "" {
 		t.Skip("VPC_ID is not set")
 	}
 
+	terraformStateBucket := os.Getenv("TERRAFORM_STATE_BUCKET")
+	if terraformStateBucket == "" {
+		t.Skip("TERRAFORM_STATE_BUCKET is not set")
+	}
+
+	terraformStateKey := os.Getenv("TERRAFORM_STATE_KEY")
+	if terraformStateKey == "" {
+		t.Skip("TERRAFORM_STATE_KEY is not set")
+	}
+
+	eksAccessPrincipalArn := os.Getenv("EKS_ACCESS_PRINCIPAL_ARN")
+	if eksAccessPrincipalArn == "" {
+		t.Skip("EKS_ACCESS_PRINCIPAL_ARN is not set")
+	}
+
 	h := testkit.New(t,
 		testkit.Providers(
 			&testkit.TerraformProvider{
 				WorkspacePath: "testdata/terraform",
 				Vars: map[string]string{
-					"prefix": "kibertas-fluentd-",
+					"prefix":                   prefix,
+					"region":                   "ap-northeast-1",
+					"vpc_id":                   vpcID,
+					"eks_access_principal_arn": eksAccessPrincipalArn,
+				},
+				BackendConfig: map[string]string{
+					"bucket": terraformStateBucket,
+					"key":    terraformStateKey,
 					"region": "ap-northeast-1",
-					"vpc_id": vpcID,
 				},
 			},
 			&testkit.KubectlProvider{},
@@ -45,6 +73,11 @@ func TestFluentE2E(t *testing.T) {
 	kc := h.KubernetesCluster(t)
 	s3Bucket := h.S3Bucket(t)
 	ns := h.KubernetesNamespace(t, testkit.KubeconfigPath(kc.KubeconfigPath))
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Logf("KUBECONFIG=%s", kc.KubeconfigPath)
+		}
+	})
 
 	k := testkit.NewKubernetes(kc.KubeconfigPath)
 	testkit.PollUntil(t, func() bool {
@@ -54,6 +87,25 @@ func TestFluentE2E(t *testing.T) {
 	helm := testkit.NewHelm(kc.KubeconfigPath)
 	helm.AddRepo(t, "chatwork", "https://chatwork.github.io/charts")
 
+	// We need to create a pod to alter the /var/log/fluentd-s3 directory
+	// because the fluentd pod cannot create the directory.
+	// Note that the fluentd pod uses:
+	//   uid=999(fluent) gid=999(fluent) groups=999(fluent)
+	kctl := testkit.NewKubectl(kc.KubeconfigPath)
+	podYamlFile, err := filepath.Abs(filepath.Join("testdata", "fluentd-alter-log-dir.pod.yaml"))
+	require.NoError(t, err)
+	require.FileExists(t, podYamlFile)
+	kctl.Capture(t,
+		"create", "-f", podYamlFile,
+	)
+	t.Cleanup(func() {
+		kctl.Capture(t, "delete", "-f", podYamlFile)
+	})
+
+	testkit.PollUntil(t, func() bool {
+		return strings.Contains(kctl.Capture(t, "get", "pod", "fluentd-alter-log-dir"), "Completed")
+	}, 30*time.Second)
+
 	fluentdNs := "default"
 	logsPath := "logs"
 	helm.UpgradeOrInstall(t, "fluentd", "chatwork/fluentd", func(hc *testkit.HelmConfig) {
@@ -71,7 +123,7 @@ func TestFluentE2E(t *testing.T) {
 				"daemonset.conf": `<source>
   @type tail
   path /var/log/containers/*.log
-  pos_file /var/log/containers.log.pos
+  pos_file /var/log/fluentd/containers.log.pos
   tag kube.*
   exclude_path ["/var/log/containers/fluent*"]
   read_from_head true
@@ -108,7 +160,6 @@ func TestFluentE2E(t *testing.T) {
 
 	fluentdClusterRoleBindingName := "fluentd-cluster-admin-binding"
 
-	kctl := testkit.NewKubectl(kc.KubeconfigPath)
 	defer func() {
 		if h.CleanupNeeded(t.Failed()) {
 			kctl.Capture(t, "delete", "clusterrolebinding", fluentdClusterRoleBindingName)

cmd/fluent/testdata/fluentd-alter-log-dir.pod.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: fluentd-alter-log-dir
+spec:
+  containers:
+  - name: aws-cli
+    image: amazon/aws-cli
+    command: ["sh", "-c", "mkdir -p /var/log/fluentd-s3 && chown -R 999:999 /var/log/fluentd-s3 && mkdir -p /var/log/fluentd && chown -R 999:999 /var/log/fluentd"]
+    volumeMounts:
+    - name: fluentd-log-dir
+      mountPath: /var/log/fluentd-s3
+  volumes:
+  - name: fluentd-log-dir
+    hostPath:
+      path: /var/log
+      type: DirectoryOrCreate
+  restartPolicy: Never

cmd/fluent/testdata/terraform/main.tf

@@ -6,9 +6,9 @@
 // - EKS cluster
 
 // Usage:
-//   terraform init
-//   terraform plan -var vpc_id=$VPC_ID -var region=ap-northeast-1 -var prefix=kibertas-fluentd-
-//   terraform apply -var vpc_id=$VPC_ID -var region=ap-northeast-1 -var prefix=kibertas-fluentd-
+//   terraform init -var vpc_id=$VPC_ID -var region=ap-northeast-1 -var prefix=$PREFIX -backend-config bucket=$TERRAFORM_STATE_BUCKET -backend-config key=$TERRAFORM_STATE_KEY -backend-config region=ap-northeast-1
+//   terraform plan -var vpc_id=$VPC_ID -var region=ap-northeast-1 -var prefix=$PREFIX
+//   terraform apply -var vpc_id=$VPC_ID -var region=ap-northeast-1 -var prefix=$PREFIX
 
 terraform {
   required_providers {
@@ -17,6 +17,7 @@ terraform {
       version = "~> 5.0"
     }
   }
+  backend "s3" {}
 }
 
 provider "aws" {
@@ -38,6 +39,11 @@ variable "region" {
     description = "The region to use for this example"
 }
 
+variable "eks_access_principal_arn" {
+    type = string
+    description = "The principal arn to use for alternative eks access"
+}
+
 // vpc cidr block
 data "aws_vpc" "vpc" {
     id = var.vpc_id
@@ -48,33 +54,63 @@ resource "aws_s3_bucket" "bucket" {
     force_destroy = true
 }
 
+locals {
+  service_ipv4_cidr = "10.100.0.0/16"
+}
+
 resource "aws_eks_cluster" "cluster" {
     name = "${var.prefix}-cluster"
     role_arn = aws_iam_role.cluster.arn
     vpc_config {
         subnet_ids = aws_subnet.public[*].id
         security_group_ids = [aws_security_group.cluster.id]
     }
-}
-
-// node group based on spot instances
-resource "aws_eks_node_group" "spot" {
-    cluster_name = aws_eks_cluster.cluster.name
-    node_group_name = "${var.prefix}-spot"
-    node_role_arn = aws_iam_role.node.arn
-    subnet_ids = aws_subnet.public[*].id
-    scaling_config {
-        desired_size = 1
-        max_size = 1
-        min_size = 1
+    access_config {
+        authentication_mode = "API_AND_CONFIG_MAP"
     }
-    # capacity_type = "SPOT"
-    instance_types = ["t3.large"]
-    labels = {
-        "role" = "spot"
+    kubernetes_network_config {
+      service_ipv4_cidr = local.service_ipv4_cidr
     }
 }
 
+resource "aws_eks_access_entry" "admin" {
+  cluster_name      = aws_eks_cluster.cluster.name
+  principal_arn     = var.eks_access_principal_arn
+}
+
+resource "aws_eks_access_policy_association" "admin" {
+  cluster_name = aws_eks_cluster.cluster.name
+  policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
+  principal_arn = var.eks_access_principal_arn
+  access_scope {
+    type       = "cluster"
+  }
+}
+
+module "eks_managed_node_group" {
+  source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group"
+  name = "spot"
+  cluster_name = aws_eks_cluster.cluster.name
+  cluster_version = aws_eks_cluster.cluster.version
+  create_iam_role = false
+  iam_role_arn = aws_iam_role.node.arn
+  subnet_ids = aws_subnet.public[*].id
+  cluster_primary_security_group_id = aws_security_group.cluster.id
+  vpc_security_group_ids = [aws_security_group.node.id]
+  instance_types = ["t3.large"]
+  capacity_type = "SPOT"
+  cluster_service_cidr = local.service_ipv4_cidr
+  metadata_options = {
+    http_tokens = "optional"
+    # As the default image is al2023 (like amazon-eks-node-al2023-x86_64-standard-1.31-v20250116 as of 2025/01/28),
+    # we need this for backward-compatibility.
+    http_put_response_hop_limit = 2
+  }
+  labels = {
+    "role" = "spot"
+  }
+}
+
 resource "aws_iam_role" "cluster" {
     name = "${var.prefix}-cluster"
     assume_role_policy = <<EOF
@@ -190,6 +226,38 @@ resource "aws_security_group" "cluster" {
     }
 }
 
+resource "aws_security_group" "node" {
+    name = "${var.prefix}-node"
+    vpc_id = data.aws_vpc.vpc.id
+}
+
+resource "aws_security_group_rule" "node_ingress" {
+    security_group_id = aws_security_group.node.id
+    type = "ingress"
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    source_security_group_id = aws_security_group.cluster.id
+}
+
+resource "aws_security_group_rule" "node_egress" {
+    security_group_id = aws_security_group.node.id
+    type = "egress"
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "node_ingress_self" {
+    security_group_id = aws_security_group.node.id
+    type = "ingress"
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    self = true
+}
+
 data "aws_availability_zones" "available" {
   state = "available"
 }

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/cert-manager/cert-manager v1.15.3
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/miekg/dns v1.1.62
-	github.com/mumoshu/testkit v0.10.0
+	github.com/mumoshu/testkit v0.11.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0

go.sum

@@ -147,8 +147,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/mumoshu/testkit v0.10.0 h1:F/m0/fUzgzf9dgdA4vqj0Bjcw98b4U1iSfaybYWWrVU=
-github.com/mumoshu/testkit v0.10.0/go.mod h1:UIqn/rsr4ziNdnV7rY/idcYrcpE9n19H10Xhwt8vLqk=
+github.com/mumoshu/testkit v0.11.0 h1:lzBEWkIxs6PnI/VGHiV/4L8eA+cOvYu2/LYSKnovCng=
+github.com/mumoshu/testkit v0.11.0/go.mod h1:UIqn/rsr4ziNdnV7rY/idcYrcpE9n19H10Xhwt8vLqk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=