From 457cebeb0f7275da69b98f366b56071bc452435f Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Mon, 12 Feb 2024 14:01:54 +0100 Subject: [PATCH] Add user namespaces tests Adding user namespaces tests for covering the `UsernamespaceMode` supported by the CRI. Fixes https://github.com/kubernetes-sigs/cri-tools/issues/1348 Signed-off-by: Sascha Grunert --- .github/workflows/containerd.yml | 25 ++-- pkg/framework/util.go | 7 ++ pkg/validate/container.go | 16 +++ pkg/validate/security_context_linux.go | 158 ++++++++++++++++++++++++- 4 files changed, 198 insertions(+), 8 deletions(-) diff --git a/.github/workflows/containerd.yml b/.github/workflows/containerd.yml index fec2f0ce32..0fe927205b 100644 --- a/.github/workflows/containerd.yml +++ b/.github/workflows/containerd.yml @@ -298,27 +298,38 @@ jobs: set -o errexit set -o nounset set -o pipefail + set -x - BDIR="$(mktemp -d -p $PWD)" + BDIR="/var/lib/containerd-critest" echo "containerd temp dir: ${BDIR}" - mkdir -p ${BDIR}/{root,state} + sudo mkdir -p ${BDIR}/{root,state} - cat > ${BDIR}/config.toml < ${BDIR}/config.toml < ${BDIR}/containerd-cri.log & + # UserNamespaces only work for 1.7 and main as well as crun + # For runc, see: + # - https://github.com/opencontainers/runc/issues/4114 + # - https://github.com/opencontainers/runc/pull/3963 + SKIP= + if [[ "${{ format(matrix.version, '/') }}" == "release/1.6" || "${{matrix.runc}}" == runc ]]; then + SKIP=--ginkgo.skip=UserNamespaces + fi + + sudo PATH=$PATH bash -c "/usr/local/bin/containerd -a ${BDIR}/c.sock -root ${BDIR}/root -state ${BDIR}/state -log-level debug &> ${BDIR}/containerd-cri.log &" sudo /usr/local/bin/ctr -a ${BDIR}/c.sock version sudo /usr/local/sbin/runc --version + sudo mount - sudo -E PATH=$PATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 - TEST_RC=$? + TEST_RC=0 + sudo -E PATH=$PATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 ${SKIP} || TEST_RC=$? test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log sudo pkill containerd echo "CONTD_CRI_DIR=$BDIR" >> $GITHUB_ENV diff --git a/pkg/framework/util.go b/pkg/framework/util.go index ff875db95e..9e7213915f 100644 --- a/pkg/framework/util.go +++ b/pkg/framework/util.go @@ -220,6 +220,13 @@ func RunPodSandbox(c internalapi.RuntimeService, config *runtimeapi.PodSandboxCo return podID } +// RunPodSandboxError runs a PodSandbox and expects an error. +func RunPodSandboxError(c internalapi.RuntimeService, config *runtimeapi.PodSandboxConfig) string { + podID, err := c.RunPodSandbox(context.TODO(), config, TestContext.RuntimeHandler) + Expect(err).To(HaveOccurred()) + return podID +} + // CreatePodSandboxForContainer creates a PodSandbox for creating containers. func CreatePodSandboxForContainer(c internalapi.RuntimeService) (string, *runtimeapi.PodSandboxConfig) { podSandboxName := "create-PodSandbox-for-container-" + NewUUID() diff --git a/pkg/validate/container.go b/pkg/validate/container.go index 4cc323fb8e..871d1b7005 100644 --- a/pkg/validate/container.go +++ b/pkg/validate/container.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "time" @@ -629,6 +630,21 @@ func verifyLogContents(podConfig *runtimeapi.PodSandboxConfig, logPath string, l Expect(found).To(BeTrue(), "expected log %q (stream=%q) not found in logs %+v", log, stream, msgs) } +// verifyLogContentsRe verifies the contents of container log using the provided regular expression pattern. +func verifyLogContentsRe(podConfig *runtimeapi.PodSandboxConfig, logPath string, pattern string, stream streamType) { + By("verify log contents using regex pattern") + msgs := parseLogLine(podConfig, logPath) + + found := false + for _, msg := range msgs { + if matched, _ := regexp.MatchString(pattern, msg.log); matched && msg.stream == stream { + found = true + break + } + } + Expect(found).To(BeTrue(), "expected log pattern %q (stream=%q) to match logs %+v", pattern, stream, msgs) +} + // listContainerStatsForID lists container for containerID. func listContainerStatsForID(c internalapi.RuntimeService, containerID string) *runtimeapi.ContainerStats { By("List container stats for containerID: " + containerID) diff --git a/pkg/validate/security_context_linux.go b/pkg/validate/security_context_linux.go index 8ebe496573..cf80d55a3d 100644 --- a/pkg/validate/security_context_linux.go +++ b/pkg/validate/security_context_linux.go @@ -842,6 +842,100 @@ var _ = framework.KubeDescribe("Security Context", func() { matchContainerOutput(podConfig, containerName, "Effective uid: 0\n") }) }) + + Context("UserNamespaces", func() { + var ( + podName string + defaultMapping = []*runtimeapi.IDMapping{{ + ContainerId: 0, + HostId: 1000, + Length: 100000, + }} + ) + + BeforeEach(func() { + podName = "user-namespaces-pod-" + framework.NewUUID() + }) + + It("runtime should support NamespaceMode_POD", func() { + namespaceOption := &runtimeapi.NamespaceOption{ + UsernsOptions: &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_POD, + Uids: defaultMapping, + Gids: defaultMapping, + }, + } + + hostLogPath, podLogPath := createLogTempDir(podName) + defer os.RemoveAll(hostLogPath) + podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath) + containerName := runUserNamespaceContainer(rc, ic, podID, podConfig) + + matchContainerOutputRe(podConfig, containerName, `\s+0\s+1000\s+100000\n`) + }) + + It("runtime should support NamespaceMode_NODE", func() { + namespaceOption := &runtimeapi.NamespaceOption{ + UsernsOptions: &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_NODE, + }, + } + + hostLogPath, podLogPath := createLogTempDir(podName) + defer os.RemoveAll(hostLogPath) + podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath) + containerName := runUserNamespaceContainer(rc, ic, podID, podConfig) + + // 4294967295 means that the entire range is available + matchContainerOutputRe(podConfig, containerName, `\s+0\s+0\s+4294967295\n`) + }) + + It("runtime should fail if more than one mapping provided", func() { + wrongMapping := []*runtimeapi.IDMapping{{ + ContainerId: 0, + HostId: 1000, + Length: 100000, + }, { + ContainerId: 0, + HostId: 2000, + Length: 100000, + }} + usernsOptions := &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_POD, + Uids: wrongMapping, + Gids: wrongMapping, + } + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + + It("runtime should fail if container ID 0 is not mapped", func() { + mapping := []*runtimeapi.IDMapping{{ + ContainerId: 1, + HostId: 1000, + Length: 100000, + }} + usernsOptions := &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_POD, + Uids: mapping, + Gids: mapping, + } + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + + It("runtime should fail with NamespaceMode_CONTAINER", func() { + usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_CONTAINER} + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + + It("runtime should fail with NamespaceMode_TARGET", func() { + usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_TARGET} + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + }) }) // matchContainerOutput matches log line in container logs. @@ -850,6 +944,12 @@ func matchContainerOutput(podConfig *runtimeapi.PodSandboxConfig, name, output s verifyLogContents(podConfig, fmt.Sprintf("%s.log", name), output, stdoutType) } +// matchContainerOutputRe matches log line in container logs using the provided regular expression pattern. +func matchContainerOutputRe(podConfig *runtimeapi.PodSandboxConfig, name, pattern string) { + By("check container output") + verifyLogContentsRe(podConfig, fmt.Sprintf("%s.log", name), pattern, stdoutType) +} + // createRunAsUserContainer creates the container with specified RunAsUser in ContainerConfig. func createRunAsUserContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string) (string, string) { By("create RunAsUser container") @@ -946,7 +1046,8 @@ func createNamespacePodSandbox(rc internalapi.RuntimeService, podSandboxNamespac uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() config := &runtimeapi.PodSandboxConfig{ - Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), + Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), + DnsConfig: &runtimeapi.DNSConfig{}, Linux: &runtimeapi.LinuxPodSandboxConfig{ SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ NamespaceOptions: podSandboxNamespace, @@ -1281,3 +1382,58 @@ func checkSetHostname(rc internalapi.RuntimeService, containerID string, setable Expect(err).To(HaveOccurred(), msg) } } + +func runUserNamespaceContainer( + rc internalapi.RuntimeService, + ic internalapi.ImageManagerService, + podID string, + podConfig *runtimeapi.PodSandboxConfig, +) string { + By("create user namespaces container") + containerName := "user-namespaces-container-" + framework.NewUUID() + containerConfig := &runtimeapi.ContainerConfig{ + Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), + Image: &runtimeapi.ImageSpec{ + Image: framework.TestContext.TestImageList.DefaultTestContainerImage, + UserSpecifiedImage: framework.TestContext.TestImageList.DefaultTestContainerImage, + }, + Command: []string{"cat", "/proc/self/uid_map"}, + LogPath: fmt.Sprintf("%s.log", containerName), + Linux: &runtimeapi.LinuxContainerConfig{ + SecurityContext: &runtimeapi.LinuxContainerSecurityContext{ + NamespaceOptions: podConfig.Linux.SecurityContext.NamespaceOptions, + }, + }, + } + + containerID := createContainerWithExpectation(rc, ic, containerConfig, podID, podConfig, true) + startContainer(rc, containerID) + + Eventually(func() runtimeapi.ContainerState { + return getContainerStatus(rc, containerID).State + }, time.Minute, time.Second*4).Should(Equal(runtimeapi.ContainerState_CONTAINER_EXITED)) + + return containerName +} + +func runUserNamespacePodWithError( + rc internalapi.RuntimeService, + podName string, + usernsOptions *runtimeapi.UserNamespace, +) { + uid := framework.DefaultUIDPrefix + framework.NewUUID() + namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + config := &runtimeapi.PodSandboxConfig{ + Metadata: framework.BuildPodSandboxMetadata(podName, uid, namespace, framework.DefaultAttempt), + Linux: &runtimeapi.LinuxPodSandboxConfig{ + SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ + NamespaceOptions: &runtimeapi.NamespaceOption{ + UsernsOptions: usernsOptions, + }, + }, + }, + Labels: framework.DefaultPodLabels, + } + + framework.RunPodSandboxError(rc, config) +}