diff --git a/docs/features/pouch_with_rich_container.md b/docs/features/pouch_with_rich_container.md index 83a516723..619304a54 100644 --- a/docs/features/pouch_with_rich_container.md +++ b/docs/features/pouch_with_rich_container.md @@ -14,23 +14,87 @@ Operators have a sacred duty to guard normal running of the applications. For th ## Get started -Users can start rich container mode in Pouch quite easily. Provided that we need to running an ordinary image in rich container mode via Pouch, there are only two flags we may add: `--rich` and `--initscript`. Here are more description about both flags: +Users can start rich container mode in Pouch quite easily. Provided that we need to running an ordinary image in rich container mode via Pouch, there are only two flags we may add: `--rich`,`--rich-mode`and `--initscript`. Here are more description about both flags: * `--rich`: identifies whether to switch on rich container mode or not. This flag has a type of `boolean`, and the default value is `false`. +* `--rich-mode`: select which manner to init container, currenly systemd, /sbin/init and dumb-init are supported. By default, it is dumb-init. * `--initscript`: identifies initial script executed in container. The script will be executed before entrypoint or command. Sometimes, it is called prestart hook. Lots of work can be done in this prestart hook, such as environment checking, environment preparation, network routes preparation, all kinds of agent settings, security setting and so on. This initscript may fail and user gets an related error message, if pouch daemon cannot find this initscript in container's filesystem which is provided by the rootfs constructed from image and potential mount volumes actually outside the container. If initscript works fine, the control of container process would be taken over by process pid 1, mainly `/sbin/init` or `dumbinit`. In fact, pouch team plans to add another flag `--initcmd` to make users input prestart hook. Actually it is a simplified one of `--initscript`. Meanwhile it brings more convenience than `--initscript`. `--initcmd` can set any command as user's wish, and things do not need to be located in image in advance. We can say command is decoupled with image. But for `--initscript`, script file must be located in image first. It is some kind of coupling. -Here is a really simple example for rich container mode: - -``` shell -pouch run --rich --initscript /home/root/startup.sh richapp:v1 -``` - If user specifies `--rich` flag and no `--initscript` flag is provided, rich container mode will still be enbaled, but no initscript will be executed. If `-rich` flag misses in command line, while `--initscript` is there, Pouch CLI or pouch daemon will return an error to show that `--initscipt` can only be used along with `--rich` flag. If a container is running with `--rich` flag, then every start or restart of this container will trigger the corresponding initscipt if there is any. +### Using dumb-init + +Here is a simple example for rich container mode using dumb-init to init contaienr: + +1. Insatll dumb-init as following: + + ```shell + # wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64 + # chmod +x /usr/bin/dumb-init + + ``` +2. Run a container with rich mode: + + ```shell + #pouch run -d --rich --rich-mode dumb-init registry.hub.docker.com/library/busybox:latest sleep 10000 + f76ac1e49e9407caf5ad33c8988b44ff3690c12aa98f7faf690545b16f2a5cbd + + #pouch exec f76ac1e49e9407caf5ad33c8988b44ff3690c12aa98f7faf690545b16f2a5cbd ps -ef + PID USER TIME COMMAND + 1 root 0:00 /usr/bin/dumb-init -- sleep 10000 + 7 root 0:00 sleep 10000 + 8 root 0:00 ps -ef + + ``` + +### Using systemd or sbin-init +In order to use systemd or /sbin/init to init container, please make sure install them on image. +As shown below, centos image has both of them. +Also `--privileged` is required in this situation. An example of systemd and sbin-init is as following: + +``` + #cat /tmp/1.sh + #! /bin/sh + echo $(cat) >/tmp/xxx + + #pouch run -d -v /tmp:/tmp --privileged --rich --rich-mode systemd --initscript /tmp/1.sh registry.hub.docker.com/library/centos:latest /usr/bin/sleep 10000 + 3054125e44443fd5ee9190ee49bbca0a842724f5305cb05df49f84fd7c901d63 + + #pouch exec 3054125e44443fd5ee9190ee49bbca0a842724f5305cb05df49f84fd7c901d63 ps aux + USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND + root 1 7.4 0.0 42968 3264 ? Ss 05:29 0:00 /usr/lib/systemd/systemd + root 17 0.0 0.0 10752 756 ? Ss 05:29 0:00 /usr/lib/systemd/systemd-readahead collect + root 18 3.2 0.0 32740 2908 ? Ss 05:29 0:00 /usr/lib/systemd/systemd-journald + root 34 0.0 0.0 22084 1456 ? Ss 05:29 0:00 /usr/lib/systemd/systemd-logind + root 36 0.0 0.0 7724 608 ? Ss 05:29 0:00 /usr/bin/sleep 10000 + dbus 37 0.0 0.0 24288 1604 ? Ss 05:29 0:00 /bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation + root 45 0.0 0.0 47452 1676 ? Rs 05:29 0:00 ps aux + + #cat /tmp/xxx + {"ociVersion":"1.0.0","id":"3054125e44443fd5ee9190ee49bbca0a842724f5305cb05df49f84fd7c901d63","status":"","pid":125745,"bundle":"/var/lib/pouch/containerd/state/io.containerd.runtime.v1.linux/default/3054125e44443fd5ee9190ee49bbca0a842724f5305cb05df49f84fd7c901d63"} + + #pouch run -d -v /tmp:/tmp --privileged --rich --rich-mode sbin-init --initscript /tmp/1.sh registry.hub.docker.com/library/centos:latest /usr/bin/sleep 10000 + c5b5eef81749ce00fb68a59ee623777bfecc8e07c617c0601cc56e4ae8b1e69f + + #pouch exec c5b5eef81749ce00fb68a59ee623777bfecc8e07c617c0601cc56e4ae8b1e69f ps aux + USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND + root 1 7.4 0.0 42968 3260 ? Ss 05:30 0:00 /sbin/init + root 17 0.0 0.0 10752 752 ? Ss 05:30 0:00 /usr/lib/systemd/systemd-readahead collect + root 20 3.2 0.0 32740 2952 ? Ss 05:30 0:00 /usr/lib/systemd/systemd-journald + root 34 0.0 0.0 22084 1452 ? Ss 05:30 0:00 /usr/lib/systemd/systemd-logind + root 35 0.0 0.0 7724 612 ? Ss 05:30 0:00 /usr/bin/sleep 10000 + dbus 36 0.0 0.0 24288 1608 ? Ss 05:30 0:00 /bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation + root 45 0.0 0.0 47452 1676 ? Rs 05:30 0:00 ps aux + + #cat /tmp/xxx + {"ociVersion":"1.0.0","id":"c5b5eef81749ce00fb68a59ee623777bfecc8e07c617c0601cc56e4ae8b1e69f","status":"","pid":127183,"bundle":"/var/lib/pouch/containerd/state/io.containerd.runtime.v1.linux/default/c5b5eef81749ce00fb68a59ee623777bfecc8e07c617c0601cc56e4ae8b1e69f"} + +``` + ## Underlying Implementation Before learning underlying implementation we shall take a brief review of `systemd`, `entrypoint` and `cmd`. In addition, prestart hook is executed by runC. @@ -43,4 +107,4 @@ To be added `initscript` is to be added. -`runc` is a CLI tool for spawning and running containers according to the OCI specification. \ No newline at end of file +`runc` is a CLI tool for spawning and running containers according to the OCI specification. diff --git a/hack/make.sh b/hack/make.sh index 394d99b4c..d0f35f875 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -60,6 +60,13 @@ function install_pouch () fi } +# Install dumb-init by downloading the binary. +function install_dumb_init +{ + wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64 || return 1 + chmod +x /usr/bin/dumb-init +} + function target() { case $1 in @@ -78,6 +85,8 @@ function target() env PATH=$GOROOT/bin:$PATH $SOURCEDIR/hack/cri-test/test-cri.sh ;; integration-test) + + install_dumb_init || echo "Warning: dumb-init install failed! rich container related tests will be skipped" docker run --rm -v $(pwd):$SOURCEDIR $IMAGE bash -c "cd test && go test -c -o integration-test" if [[ $SOURCEDIR != $DIR ]];then diff --git a/test/cli_rich_container_test.go b/test/cli_rich_container_test.go new file mode 100644 index 000000000..1847869ea --- /dev/null +++ b/test/cli_rich_container_test.go @@ -0,0 +1,235 @@ +package main + +import ( + "errors" + "fmt" + "runtime" + "strings" + + "github.com/alibaba/pouch/apis/types" + "github.com/alibaba/pouch/test/command" + "github.com/alibaba/pouch/test/environment" + + "encoding/json" + "github.com/go-check/check" + "github.com/gotestyourself/gotestyourself/icmd" +) + +// PouchRichContainerSuite is the test suite fo rich container related CLI. +type PouchRichContainerSuite struct{} + +func init() { + check.Suite(&PouchRichContainerSuite{}) +} + +var centosImage string + +// SetUpSuite does common setup in the beginning of each test suite. +func (suite *PouchRichContainerSuite) SetUpSuite(c *check.C) { + SkipIfFalse(c, environment.IsLinux) + SkipIfFalse(c, environment.IsRuncVersionSupportRichContianer) + + command.PouchRun("pull", busyboxImage).Assert(c, icmd.Success) + + // Use image from AliYun on AliOS. + if environment.IsAliKernel() { + centosImage = "reg.docker.alibaba-inc.com/alibase/alios7u2:latest" + } else { + centosImage = "registry.hub.docker.com/library/centos:latest" + } + command.PouchRun("pull", centosImage).Assert(c, icmd.Success) +} + +// TearDownSuite does common cleanup in the end of each test suite. +func (suite *PouchRichContainerSuite) TearDownSuite(c *check.C) { + SkipIfFalse(c, environment.IsLinux) + SkipIfFalse(c, environment.IsRuncVersionSupportRichContianer) + + command.PouchRun("rmi", centosImage) +} + +// isFileExistsInImage checks if the file exists in given image. +func isFileExistsInImage(image string, file string, cname string) (bool, error) { + if image == "" || file == "" || cname == "" { + return false, errors.New("input args is nil") + } + + // check the existence of /sbin/init in image + expect := icmd.Expected{ + ExitCode: 0, + Out: "Access", + } + err := command.PouchRun("run", "--name", cname, image, "stat", file).Compare(expect) + command.PouchRun("rm", "-f", cname) + + return err == nil, nil +} + +// checkPidofProcessIsOne checks the process of pid 1 is expected. +func checkPidofProcessIsOne(cname string, p string) bool { + expect := icmd.Expected{ + ExitCode: 0, + Out: "1", + } + + err := command.PouchRun("exec", cname, + "ps", "-ef", "| grep", p, "| awk '{print $1}'").Compare(expect) + if err != nil { + fmt.Printf("err=%s\n", err) + } + return err == nil +} + +// checkPPid checks the ppid of process is expected. +func checkPPid(cname string, p string, ppid string) bool { + expect := icmd.Expected{ + ExitCode: 0, + Out: ppid, + } + + err := command.PouchRun("exec", cname, + "ps", "-ef", "|grep", p, "|awk '{print $3}'").Compare(expect) + if err != nil { + fmt.Printf("err=%s\n", err) + } + + return err == nil +} + +// checkInitScriptWorks +func checkInitScriptWorks(c *check.C, cname string, image string, richmode string) { + // TODO: Use bash script to get the stdin may have error, need to find a better way. + +} + +// TestRichContainerDumbInitWorks check the dumb-init works. +func (suite *PouchRichContainerSuite) TestRichContainerDumbInitWorks(c *check.C) { + SkipIfFalse(c, environment.IsDumbInitExist) + pc, _, _, _ := runtime.Caller(0) + tmpname := strings.Split(runtime.FuncForPC(pc).Name(), ".") + var funcname string + for i := range tmpname { + funcname = tmpname[i] + } + + command.PouchRun("run", "-d", "--rich", "--rich-mode", "dumb-init", "--name", funcname, + busyboxImage, "sleep", "10000").Assert(c, icmd.Success) + + output := command.PouchRun("inspect", funcname).Stdout() + result := &types.ContainerJSON{} + if err := json.Unmarshal([]byte(output), result); err != nil { + c.Errorf("failed to decode inspect output: %v", err) + } + c.Assert(result.Config.Rich, check.Equals, true) + c.Assert(result.Config.RichMode, check.Equals, "dumb-init") + + c.Assert(checkPidofProcessIsOne(funcname, "dumb-init"), check.Equals, true) + + // stop and start could work well. + command.PouchRun("stop", funcname).Assert(c, icmd.Success) + command.PouchRun("start", funcname).Assert(c, icmd.Success) + c.Assert(checkPidofProcessIsOne(funcname, "dumb-init"), check.Equals, true) + + // pause and unpause + command.PouchRun("pause", funcname).Assert(c, icmd.Success) + command.PouchRun("unpause", funcname).Assert(c, icmd.Success) + c.Assert(checkPidofProcessIsOne(funcname, "dumb-init"), check.Equals, true) + + command.PouchRun("rm", "-f", funcname) +} + +// TestRichContainerWrongArgs check the wrong args of rich container. +func (suite *PouchRichContainerSuite) TestRichContainerDumbInitWrongArgs(c *check.C) { + SkipIfFalse(c, environment.IsDumbInitExist) + + // TODO + // Don't add '--rich' when use other rich container related options should fail. + +} + +// TestRichContainerSbinInitWorks check the initd works. +func (suite *PouchRichContainerSuite) TestRichContainerInitdWorks(c *check.C) { + pc, _, _, _ := runtime.Caller(0) + tmpname := strings.Split(runtime.FuncForPC(pc).Name(), ".") + var funcname string + for i := range tmpname { + funcname = tmpname[i] + } + + ok, _ := isFileExistsInImage(centosImage, "/sbin/init", "checkinit") + if !ok { + c.Skip("/sbin/init doesn't exist in test image") + } + + // --privileged is MUST required + command.PouchRun("run", "-d", "--privileged", "--rich", "--rich-mode", "sbin-init", + "--name", funcname, centosImage, "/usr/bin/sleep 10000").Assert(c, icmd.Success) + + output := command.PouchRun("inspect", funcname).Stdout() + result := &types.ContainerJSON{} + if err := json.Unmarshal([]byte(output), result); err != nil { + c.Errorf("failed to decode inspect output: %v", err) + } + c.Assert(result.Config.Rich, check.Equals, true) + c.Assert(result.Config.RichMode, check.Equals, "sbin-init") + + c.Assert(checkPidofProcessIsOne(funcname, "/sbin/init"), check.Equals, true) + c.Assert(checkPPid(funcname, "sleep", "1"), check.Equals, true) + + // stop and start could work well. + command.PouchRun("stop", funcname).Assert(c, icmd.Success) + command.PouchRun("start", funcname).Assert(c, icmd.Success) + c.Assert(checkPidofProcessIsOne(funcname, "/sbin/init"), check.Equals, true) + c.Assert(checkPPid(funcname, "sleep", "1"), check.Equals, true) + + // pause and unpause + command.PouchRun("pause", funcname).Assert(c, icmd.Success) + command.PouchRun("unpause", funcname).Assert(c, icmd.Success) + c.Assert(checkPidofProcessIsOne(funcname, "/sbin/init"), check.Equals, true) + c.Assert(checkPPid(funcname, "sleep", "1"), check.Equals, true) + + command.PouchRun("rm", "-f", funcname) +} + +// TestRichContainerSystemdWorks check the systemd works. +func (suite *PouchRichContainerSuite) TestRichContainerSystemdWorks(c *check.C) { + pc, _, _, _ := runtime.Caller(0) + tmpname := strings.Split(runtime.FuncForPC(pc).Name(), ".") + var funcname string + for i := range tmpname { + funcname = tmpname[i] + } + + ok, _ := isFileExistsInImage(centosImage, "/usr/lib/systemd/systemd", "checksysd") + if !ok { + c.Skip("/usr/lib/systemd/systemd doesn't exist in test image") + } + + command.PouchRun("run", "-d", "--privileged", "--rich", "--rich-mode", "systemd", + "--name", funcname, centosImage, "/usr/bin/sleep 1000").Assert(c, icmd.Success) + + output := command.PouchRun("inspect", funcname).Stdout() + result := &types.ContainerJSON{} + if err := json.Unmarshal([]byte(output), result); err != nil { + c.Errorf("failed to decode inspect output: %v", err) + } + c.Assert(result.Config.Rich, check.Equals, true) + c.Assert(result.Config.RichMode, check.Equals, "systemd") + + c.Assert(checkPidofProcessIsOne(funcname, "/usr/lib/systemd/systemd"), check.Equals, true) + c.Assert(checkPPid(funcname, "sleep", "1"), check.Equals, true) + + // stop and start could work well. + command.PouchRun("stop", funcname).Assert(c, icmd.Success) + command.PouchRun("start", funcname).Assert(c, icmd.Success) + c.Assert(checkPidofProcessIsOne(funcname, "/usr/lib/systemd/systemd"), check.Equals, true) + c.Assert(checkPPid(funcname, "sleep", "1"), check.Equals, true) + + // pause and unpause + command.PouchRun("pause", funcname).Assert(c, icmd.Success) + command.PouchRun("unpause", funcname).Assert(c, icmd.Success) + c.Assert(checkPidofProcessIsOne(funcname, "/usr/lib/systemd/systemdd"), check.Equals, true) + c.Assert(checkPPid(funcname, "sleep", "1"), check.Equals, true) + + command.PouchRun("rm", "-f", funcname) +} diff --git a/test/environment/env.go b/test/environment/env.go index 856864ff5..186a3b184 100644 --- a/test/environment/env.go +++ b/test/environment/env.go @@ -1,6 +1,7 @@ package environment import ( + "os" "runtime" "github.com/alibaba/pouch/pkg/utils" @@ -34,3 +35,20 @@ func IsAliKernel() bool { } return false } + +// IsDumbInitExist checks if the dumb-init binary exists on host. +func IsDumbInitExist() bool { + if _, err := os.Stat("/usr/bin/dumb-init"); err != nil && os.IsNotExist(err) { + return false + } + return true +} + +// IsRuncVersionSupportRichContianer checks if the version of runc supports rich container. +func IsRuncVersionSupportRichContianer() bool { + cmd := "runc -v|grep 1.0.0-rc4-1" + if icmd.RunCommand("bash", "-c", cmd).ExitCode == 0 { + return true + } + return false +}