diff --git a/Makefile b/Makefile index 99eb20775d..5d6a9a2788 100644 --- a/Makefile +++ b/Makefile @@ -346,7 +346,7 @@ docker-tests: load-basic $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME)-dcache,--fdlimit=2000 --dcache=100) # Used by TestDentryCacheLimit. @$(call install_runtime,$(RUNTIME)-host-uds,--host-uds=all) # Used by TestHostSocketConnect. @$(call install_runtime,$(RUNTIME)-overlay,--overlay2=all:self) # Used by TestOverlay*. - @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test) + @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test //test/e2e:runtime_in_docker_test) .PHONY: docker-tests plugin-network-tests: load-basic $(RUNTIME_BIN) diff --git a/images/basic/integrationtest/Dockerfile b/images/basic/integrationtest/Dockerfile index 8b237bf659..5d4789a159 100644 --- a/images/basic/integrationtest/Dockerfile +++ b/images/basic/integrationtest/Dockerfile @@ -4,7 +4,8 @@ WORKDIR /root COPY . . RUN chmod +x *.sh -RUN apt-get update && apt-get install -y gcc iputils-ping iproute2 +RUN apt-get update && apt-get install -y \ + build-essential iputils-ping iproute2 iptables # Compilation Steps. RUN gcc -O2 -o test_copy_up test_copy_up.c @@ -12,4 +13,12 @@ RUN gcc -O2 -o test_rewinddir test_rewinddir.c RUN gcc -O2 -o link_test link_test.c RUN gcc -O2 -o test_sticky test_sticky.c RUN gcc -O2 -o host_fd host_fd.c -RUN gcc -O2 -o host_connect host_connect.c \ No newline at end of file +RUN gcc -O2 -o host_connect host_connect.c + +# Add nonprivileged regular user named "nonroot". +RUN groupadd --gid 1337 nonroot && \ + useradd --uid 1337 --gid 1337 \ + --create-home \ + --shell $(which bash) \ + --password '' \ + nonroot diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index a83f5d322c..05fea5abb8 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -116,6 +116,9 @@ type RunOpts struct { Devices []container.DeviceMapping + // SecurityOpts are security options to set on the container. + SecurityOpts []string + // sniffGPUOpts, if set, sets the rules for GPU sniffing during this test. // Must be set via `RunOpts.SniffGPU`. sniffGPUOpts *SniffGPUOpts @@ -347,6 +350,7 @@ func (c *Container) hostConfig(r RunOpts) *container.HostConfig { CapAdd: r.CapAdd, CapDrop: r.CapDrop, Privileged: r.Privileged, + SecurityOpt: r.SecurityOpts, ReadonlyRootfs: r.ReadOnly, NetworkMode: container.NetworkMode(r.NetworkMode), Resources: container.Resources{ diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 9ce631302d..4106775c82 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -52,6 +52,26 @@ go_test( ], ) +go_test( + name = "runtime_in_docker_test", + size = "large", + srcs = [ + "runtime_in_docker_test.go", + ], + library = ":integration", + tags = [ + # Requires docker and runsc to be configured before the test runs. + "local", + "manual", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//pkg/test/testutil", + "@com_github_docker_docker//api/types/mount:go_default_library", + ], +) + go_library( name = "integration", srcs = ["integration.go"], diff --git a/test/e2e/integration_runtime_test.go b/test/e2e/integration_runtime_test.go index f894bce32f..fc623e7299 100644 --- a/test/e2e/integration_runtime_test.go +++ b/test/e2e/integration_runtime_test.go @@ -104,7 +104,7 @@ func TestDentryCacheLimit(t *testing.T) { } } - // Run the container. Open a bunch of files simutaneously and sleep a bit + // Run the container. Open a bunch of files simultaneously and sleep a bit // to give time for everything to start. We shouldn't hit the FD limit // because the dentry cache is small. cmd := `for file in /tmp/foo/*; do (cat > "${file}") & done && sleep 10` diff --git a/test/e2e/runtime_in_docker_test.go b/test/e2e/runtime_in_docker_test.go new file mode 100644 index 0000000000..07eb9c67cd --- /dev/null +++ b/test/e2e/runtime_in_docker_test.go @@ -0,0 +1,228 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package integration provides end-to-end integration tests for runsc. +package integration + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + + "github.com/docker/docker/api/types/mount" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +// testVariant is a variant of the gVisor in Docker test. +type testVariant struct { + Name string + User string + WorkDir string + CapAdd []string + Args []string + MountCgroupfs bool +} + +// run runs the test variant. +func (test testVariant) run(ctx context.Context, logger testutil.Logger, runscPath string) (string, error) { + d := dockerutil.MakeNativeContainer(ctx, logger) + defer d.CleanUp(ctx) + opts := dockerutil.RunOpts{ + Image: "basic/integrationtest", + User: test.User, + WorkDir: test.WorkDir, + SecurityOpts: []string{ + // Disable default seccomp filter which blocks `mount(2)` and others. + "seccomp=unconfined", + + // Disable AppArmor which also blocks mounts. + "apparmor=unconfined", + + // Set correct SELinux label; this allows ptrace. + "label=type:container_engine_t", + }, + CapAdd: test.CapAdd, + Mounts: []mount.Mount{ + // Mount the runtime binary. + { + Type: mount.TypeBind, + Source: runscPath, + Target: "/runtime", + ReadOnly: true, + }, + }, + } + if test.MountCgroupfs { + opts.Mounts = append(opts.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: "/sys/fs/cgroup", + Target: "/sys/fs/cgroup", + ReadOnly: false, + }) + } + // Remove obstructed read-only masks over procfs so that `runsc` can + // mount a new instance of `procfs` in the chroot. + // TODO(gvisor.dev/issue/10944): Remove this once issue is fixed. + opts.SecurityOpts = append(opts.SecurityOpts, "--security-opt=systempaths=unconfined") + const wantMessage = "It became a jumble of words, a litany, almost a kind of glossolalia." + args := []string{ + "/runtime", + "--debug=true", + "--debug-log=/dev/stderr", + } + args = append(args, test.Args...) + args = append(args, "do", "/bin/echo", wantMessage) + logger.Logf("Running: %v", args) + got, err := d.Run(ctx, opts, args...) + got = strings.TrimSpace(got) + if err != nil { + return got, err + } + if !strings.Contains(got, wantMessage) { + return got, fmt.Errorf("did not observe substring %q in logs", wantMessage) + } + return got, nil +} + +// failureCases returns modified versions of this same test that are expected +// to fail. Verifying that these variants fail ensures that each test variant +// runs with the minimal amount of deviations from the default configuration. +func (test testVariant) failureCases() []testVariant { + failureCase := func(name string) testVariant { + copy := test + copy.Name = name + return copy + } + var failureCases []testVariant + if test.MountCgroupfs { + copy := failureCase("without cgroupfs mounted") + copy.MountCgroupfs = false + failureCases = append(failureCases, copy) + } + for i, capAdd := range test.CapAdd { + copy := failureCase(fmt.Sprintf("without capability %s", capAdd)) + copy.CapAdd = append(append([]string(nil), test.CapAdd[:i]...), test.CapAdd[i+1:]...) + failureCases = append(failureCases, copy) + } + for _, tryRemoveArg := range []string{ + "--rootless=true", + "--ignore-cgroups=true", + } { + if index := slices.Index(test.Args, tryRemoveArg); index != -1 { + copy := failureCase(fmt.Sprintf("without argument %s", tryRemoveArg)) + copy.Args = append(append([]string(nil), test.Args[:index]...), test.Args[index+1:]...) + failureCases = append(failureCases, copy) + } + } + return failureCases +} + +// TestGVisorInDocker runs `runsc` inside a non-gVisor container. +// This is used in contexts such as Dangerzone: +// https://gvisor.dev/blog/2024/09/23/safe-ride-into-the-dangerzone/ +func TestGVisorInDocker(t *testing.T) { + ctx := context.Background() + runscPath, err := dockerutil.RuntimePath() + if err != nil { + t.Fatalf("Cannot locate runtime path: %v", err) + } + for _, test := range []testVariant{ + { + Name: "Rootful", + User: "root", + CapAdd: []string{ + // Necessary to set up networking (creating veth devices). + "NET_ADMIN", + // Necessary to set up networking, which calls `ip netns add` which + // calls `mount(2)`. + "SYS_ADMIN", + }, + // Mount cgroupfs as writable, otherwise the runtime won't be able to + // set up cgroups. + MountCgroupfs: true, + }, + { + Name: "Rootful without networking", + User: "root", + CapAdd: []string{ + // "Can't run sandbox process in minimal chroot since we don't have CAP_SYS_ADMIN" + "SYS_ADMIN", + }, + Args: []string{ + "--network=none", + }, + MountCgroupfs: true, + }, + { + Name: "Rootful with host networking", + User: "root", + CapAdd: []string{ + // Necessary to set up networking (creating veth devices). + "NET_ADMIN", + // Necessary to set up networking, which calls `ip netns add` which + // calls `mount(2)`. + "SYS_ADMIN", + }, + Args: []string{ + "--network=host", + }, + MountCgroupfs: true, + }, + { + Name: "Rootful without networking and cgroupfs", + User: "root", + CapAdd: []string{ + // "Can't run sandbox process in minimal chroot since we don't have CAP_SYS_ADMIN" + "SYS_ADMIN", + }, + Args: []string{ + "--network=none", + "--ignore-cgroups=true", + }, + }, + { + Name: "Rootless", + User: "nonroot", + WorkDir: "/home/nonroot", + Args: []string{ + "--rootless=true", + }, + }, + { + Name: "Rootless without networking", + User: "nonroot", + WorkDir: "/home/nonroot", + Args: []string{ + "--rootless=true", + "--network=none", + }, + }, + } { + t.Run(test.Name, func(t *testing.T) { + if logs, err := test.run(ctx, t, runscPath); err != nil { + t.Fatalf("Error: %v; logs:\n%s", err, logs) + } + for _, failureCase := range test.failureCases() { + t.Run(failureCase.Name, func(t *testing.T) { + if logs, err := failureCase.run(ctx, t, runscPath); err == nil { + t.Fatalf("Failure case unexpectedly succeeded; logs:\n%s", logs) + } + }) + } + }) + } +}