diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go index 00c2595a6..e2aa0e9b2 100644 --- a/kola/tests/kubeadm/kubeadm.go +++ b/kola/tests/kubeadm/kubeadm.go @@ -54,8 +54,12 @@ var ( _ = c.MustSSH(controller, "/opt/bin/cilium uninstall") version := params["CiliumVersion"].(string) cidr := params["PodSubnet"].(string) - cmd := fmt.Sprintf("/opt/bin/cilium install --config enable-endpoint-routes=true --config cluster-pool-ipv4-cidr=%s --version=%s --encryption=ipsec --wait --wait-duration 1m", cidr, version) - _ = c.MustSSH(controller, cmd) + cmd := fmt.Sprintf("/opt/bin/cilium install --config enable-endpoint-routes=true --config cluster-pool-ipv4-cidr=%s --version=%s --encryption=ipsec --wait=false --restart-unmanaged-pods=false --rollback=false", cidr, version) + _, _ = c.SSH(controller, cmd) + patch := `{ grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && /opt/bin/kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true` + _ = c.MustSSH(controller, patch) + status := "/opt/bin/cilium status --wait --wait-duration 1m" + _ = c.MustSSH(controller, status) }, }, }, @@ -368,6 +372,32 @@ func setup(c cluster.TestCluster, params map[string]interface{}) (platform.Machi return nil, fmt.Errorf("unable to create etcd node: %w", err) } + v := string(c.MustSSH(etcdNode, `set -euo pipefail; grep -m 1 "^VERSION=" /usr/lib/os-release | cut -d = -f 2`)) + if v == "" { + c.Fatalf("Assertion for version string failed") + } + + version, err := semver.NewVersion(v) + if err != nil { + c.Fatalf("unable to create semver version from %s: %v", version, err) + } + + // For Cilium CNI, we enforce SELinux only for version >= 3745 because the SELinux policies update (container_t/spc_t) is not yet + // propagated through all the channels. + // The etcd node will run with enforced SELinux anyway but we want to test SELinux on the worker / master nodes. + cni, ok := params["CNI"] + if !ok { + c.Fatal("unable to get CNI value") + } + + if cni == "cilium" && version.LessThan(semver.Version{Major: 3745}) { + r := c.RuntimeConf() + if r != nil { + plog.Infof("Setting SELinux to permissive mode") + r.NoEnableSelinux = true + } + } + if err := etcd.GetClusterHealth(c, etcdNode, 1); err != nil { return nil, fmt.Errorf("unable to get etcd node health: %w", err) } diff --git a/kola/tests/kubeadm/templates.go b/kola/tests/kubeadm/templates.go index f54cfa990..2ceae6d19 100644 --- a/kola/tests/kubeadm/templates.go +++ b/kola/tests/kubeadm/templates.go @@ -403,6 +403,7 @@ EOF --config enable-endpoint-routes=true \ --config cluster-pool-ipv4-cidr={{ .PodSubnet }} \ --version={{ .CiliumVersion }} 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT + { grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true # --wait will wait for status to report success /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT {{ end }} diff --git a/kola/tests/kubeadm/testdata/master-cilium-script.sh b/kola/tests/kubeadm/testdata/master-cilium-script.sh index 72c099e77..61a1e6e5c 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-script.sh +++ b/kola/tests/kubeadm/testdata/master-cilium-script.sh @@ -91,6 +91,7 @@ EOF --config enable-endpoint-routes=true \ --config cluster-pool-ipv4-cidr=192.168.0.0/17 \ --version=v0.11.1 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT + { grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true # --wait will wait for status to report success /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT diff --git a/platform/cluster.go b/platform/cluster.go index e59b9dc50..e7adfc67b 100644 --- a/platform/cluster.go +++ b/platform/cluster.go @@ -283,8 +283,8 @@ func (bc *BaseCluster) Name() string { return bc.name } -func (bc *BaseCluster) RuntimeConf() RuntimeConfig { - return *bc.rconf +func (bc *BaseCluster) RuntimeConf() *RuntimeConfig { + return bc.rconf } func (bc *BaseCluster) ConsoleOutput() map[string]string { diff --git a/platform/machine/aws/machine.go b/platform/machine/aws/machine.go index bdd511972..d97392088 100644 --- a/platform/machine/aws/machine.go +++ b/platform/machine/aws/machine.go @@ -48,7 +48,7 @@ func (am *machine) PrivateIP() string { return *am.mach.PrivateIpAddress } -func (am *machine) RuntimeConf() platform.RuntimeConfig { +func (am *machine) RuntimeConf() *platform.RuntimeConfig { return am.cluster.RuntimeConf() } diff --git a/platform/machine/azure/machine.go b/platform/machine/azure/machine.go index 0a4c56fe4..2470f5dbb 100644 --- a/platform/machine/azure/machine.go +++ b/platform/machine/azure/machine.go @@ -45,7 +45,7 @@ func (am *machine) PrivateIP() string { return am.mach.PrivateIPAddress } -func (am *machine) RuntimeConf() platform.RuntimeConfig { +func (am *machine) RuntimeConf() *platform.RuntimeConfig { return am.cluster.RuntimeConf() } diff --git a/platform/machine/do/machine.go b/platform/machine/do/machine.go index dce8a4092..c449a2c32 100644 --- a/platform/machine/do/machine.go +++ b/platform/machine/do/machine.go @@ -44,7 +44,7 @@ func (dm *machine) PrivateIP() string { return dm.privateIP } -func (dm *machine) RuntimeConf() platform.RuntimeConfig { +func (dm *machine) RuntimeConf() *platform.RuntimeConfig { return dm.cluster.RuntimeConf() } diff --git a/platform/machine/equinixmetal/machine.go b/platform/machine/equinixmetal/machine.go index dda45855c..0a655db47 100644 --- a/platform/machine/equinixmetal/machine.go +++ b/platform/machine/equinixmetal/machine.go @@ -44,7 +44,7 @@ func (pm *machine) PrivateIP() string { return pm.privateIP } -func (pm *machine) RuntimeConf() platform.RuntimeConfig { +func (pm *machine) RuntimeConf() *platform.RuntimeConfig { return pm.cluster.RuntimeConf() } diff --git a/platform/machine/esx/machine.go b/platform/machine/esx/machine.go index ab175111c..2a7b04221 100644 --- a/platform/machine/esx/machine.go +++ b/platform/machine/esx/machine.go @@ -46,7 +46,7 @@ func (em *machine) PrivateIP() string { return em.mach.IPAddress } -func (em *machine) RuntimeConf() platform.RuntimeConfig { +func (em *machine) RuntimeConf() *platform.RuntimeConfig { return em.cluster.RuntimeConf() } diff --git a/platform/machine/external/machine.go b/platform/machine/external/machine.go index 58736f0c6..a2d6e16ea 100644 --- a/platform/machine/external/machine.go +++ b/platform/machine/external/machine.go @@ -40,7 +40,7 @@ func (pm *machine) PrivateIP() string { return pm.ipAddr } -func (pm *machine) RuntimeConf() platform.RuntimeConfig { +func (pm *machine) RuntimeConf() *platform.RuntimeConfig { return pm.cluster.RuntimeConf() } diff --git a/platform/machine/gcloud/machine.go b/platform/machine/gcloud/machine.go index ae7c605c8..00fd99a4e 100644 --- a/platform/machine/gcloud/machine.go +++ b/platform/machine/gcloud/machine.go @@ -45,7 +45,7 @@ func (gm *machine) PrivateIP() string { return gm.intIP } -func (gm *machine) RuntimeConf() platform.RuntimeConfig { +func (gm *machine) RuntimeConf() *platform.RuntimeConfig { return gm.gc.RuntimeConf() } diff --git a/platform/machine/openstack/machine.go b/platform/machine/openstack/machine.go index 823ae1afd..65a9bc6ef 100644 --- a/platform/machine/openstack/machine.go +++ b/platform/machine/openstack/machine.go @@ -103,7 +103,7 @@ func (om *machine) PrivateIP() string { return om.IP() } -func (om *machine) RuntimeConf() platform.RuntimeConfig { +func (om *machine) RuntimeConf() *platform.RuntimeConfig { return om.cluster.RuntimeConf() } diff --git a/platform/machine/qemu/machine.go b/platform/machine/qemu/machine.go index 7aa887d92..50883f734 100644 --- a/platform/machine/qemu/machine.go +++ b/platform/machine/qemu/machine.go @@ -46,7 +46,7 @@ func (m *machine) PrivateIP() string { return m.netif.DHCPv4[0].IP.String() } -func (m *machine) RuntimeConf() platform.RuntimeConfig { +func (m *machine) RuntimeConf() *platform.RuntimeConfig { return m.qc.RuntimeConf() } diff --git a/platform/machine/unprivqemu/machine.go b/platform/machine/unprivqemu/machine.go index 1eb8c6cd0..647f8f518 100644 --- a/platform/machine/unprivqemu/machine.go +++ b/platform/machine/unprivqemu/machine.go @@ -46,7 +46,7 @@ func (m *machine) PrivateIP() string { return m.privateAddr } -func (m *machine) RuntimeConf() platform.RuntimeConfig { +func (m *machine) RuntimeConf() *platform.RuntimeConfig { return m.qc.RuntimeConf() } diff --git a/platform/platform.go b/platform/platform.go index 1a53cc0fc..dd9c01e28 100644 --- a/platform/platform.go +++ b/platform/platform.go @@ -50,7 +50,7 @@ type Machine interface { PrivateIP() string // RuntimeConf returns the cluster's runtime configuration. - RuntimeConf() RuntimeConfig + RuntimeConf() *RuntimeConfig // SSHClient establishes a new SSH connection to the machine. SSHClient() (*ssh.Client, error) @@ -113,6 +113,9 @@ type Cluster interface { // IgnitionVersion returns the version of Ignition supported by the // cluster IgnitionVersion() string + + // RuntimeConf returns a pointer to the runtime configuration. + RuntimeConf() *RuntimeConfig } // Flight represents a group of Clusters within a single platform.