agent: add support for guest-hooks

Adds support for running OCI hooks within the guest. A 'drop-in'
path (guest_hook_path) is specified in the cli configuration file
and if set, the agent will look for OCI hooks in this directory
and inject them into the container life cycle.

Fixes kata-containers#348

Signed-off-by: Edward Guzman <eguzman@nvidia.com>

agent.go

@@ -29,6 +29,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	_ "github.com/opencontainers/runc/libcontainer/nsenter"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	"golang.org/x/sys/unix"
@@ -93,25 +94,27 @@ type sandboxStorage struct {
 type sandbox struct {
 	sync.RWMutex
 
-	id              string
-	hostname        string
-	containers      map[string]*container
-	channel         channel
-	network         network
-	wg              sync.WaitGroup
-	sharedPidNs     namespace
-	mounts          []string
-	subreaper       reaper
-	server          *grpc.Server
-	pciDeviceMap    map[string]string
-	deviceWatchers  map[string](chan string)
-	sharedUTSNs     namespace
-	sharedIPCNs     namespace
-	running         bool
-	noPivotRoot     bool
-	enableGrpcTrace bool
-	sandboxPidNs    bool
-	storages        map[string]*sandboxStorage
+	id                string
+	hostname          string
+	containers        map[string]*container
+	channel           channel
+	network           network
+	wg                sync.WaitGroup
+	sharedPidNs       namespace
+	mounts            []string
+	subreaper         reaper
+	server            *grpc.Server
+	pciDeviceMap      map[string]string
+	deviceWatchers    map[string](chan string)
+	sharedUTSNs       namespace
+	sharedIPCNs       namespace
+	running           bool
+	noPivotRoot       bool
+	enableGrpcTrace   bool
+	sandboxPidNs      bool
+	storages          map[string]*sandboxStorage
+	guestHooks        *specs.Hooks
+	guestHooksPresent bool
 }
 
 var agentFields = logrus.Fields{
@@ -231,6 +234,34 @@ func (s *sandbox) setSandboxStorage(path string) bool {
 	return false
 }
 
+// scanGuestHooks will search the given guestHookPath
+// for any OCI hooks
+func (s *sandbox) scanGuestHooks(guestHookPath string) {
+	s.guestHooks.Prestart = pb.FindHooks(guestHookPath, "prestart")
+	s.guestHooks.Poststart = pb.FindHooks(guestHookPath, "poststart")
+	s.guestHooks.Poststop = pb.FindHooks(guestHookPath, "poststop")
+
+	if len(s.guestHooks.Prestart) > 0 || len(s.guestHooks.Poststart) > 0 || len(s.guestHooks.Poststop) > 0 {
+		s.guestHooksPresent = true
+	}
+}
+
+// addGuestHooks will add any guest OCI hooks that were
+// found to the OCI spec
+func (s *sandbox) addGuestHooks(spec *specs.Spec) {
+	if spec == nil {
+		return
+	}
+
+	if spec.Hooks == nil {
+		spec.Hooks = &specs.Hooks{}
+	}
+
+	spec.Hooks.Prestart = append(spec.Hooks.Prestart, s.guestHooks.Prestart...)
+	spec.Hooks.Poststart = append(spec.Hooks.Poststart, s.guestHooks.Poststart...)
+	spec.Hooks.Poststop = append(spec.Hooks.Poststop, s.guestHooks.Poststop...)
+}
+
 // unSetSandboxStorage will decrement the sandbox storage
 // reference counter. If there aren't any containers using
 // that sandbox storage, this method will remove the

agent_test.go

@@ -9,15 +9,18 @@ package main
 import (
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"syscall"
 	"testing"
 
 	"google.golang.org/grpc"
 
 	pb "github.com/kata-containers/agent/protocols/grpc"
 	"github.com/opencontainers/runc/libcontainer"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/net/context"
 )
@@ -446,3 +449,52 @@ func TestGetCgroupMountsSuccessful(t *testing.T) {
 	err = syscall.Unmount(cgprocDir, 0)
 	assert.Nil(t, err, "%v", err)
 }
+
+func TestAddGuestHooks(t *testing.T) {
+	assert := assert.New(t)
+
+	hookPath, err := ioutil.TempDir("", "hooks")
+	assert.NoError(err)
+	defer os.RemoveAll(hookPath)
+
+	poststopPath := path.Join(hookPath, "poststop")
+	err = os.Mkdir(poststopPath, 0777)
+	assert.NoError(err)
+
+	dirPath := path.Join(poststopPath, "directory")
+	err = os.Mkdir(dirPath, 0777)
+	assert.NoError(err)
+
+	normalPath := path.Join(poststopPath, "normalfile")
+	f, err := os.OpenFile(normalPath, os.O_RDONLY|os.O_CREATE, 0666)
+	assert.NoError(err)
+	f.Close()
+
+	symlinkPath := path.Join(poststopPath, "symlink")
+	err = os.Link(normalPath, symlinkPath)
+	assert.NoError(err)
+
+	s := &sandbox{
+		guestHooks:        &specs.Hooks{},
+		guestHooksPresent: false,
+	}
+
+	s.scanGuestHooks(hookPath)
+	assert.False(s.guestHooksPresent)
+
+	spec := &specs.Spec{}
+	s.addGuestHooks(spec)
+	assert.True(len(spec.Hooks.Poststop) == 0)
+
+	execPath := path.Join(poststopPath, "executable")
+	f, err = os.OpenFile(execPath, os.O_RDONLY|os.O_CREATE, 0777)
+	assert.NoError(err)
+	f.Close()
+
+	s.scanGuestHooks(hookPath)
+	assert.True(s.guestHooksPresent)
+
+	s.addGuestHooks(spec)
+	assert.True(len(spec.Hooks.Poststop) == 1)
+	assert.True(strings.Contains(spec.Hooks.Poststop[0].Path, "executable"))
+}

grpc.go

@@ -523,6 +523,35 @@ func (a *agentGRPC) rollbackFailingContainerCreation(ctr *container) {
 	}
 }
 
+func (a *agentGRPC) finishCreateContainer(ctr *container, req *pb.CreateContainerRequest, config *configs.Config) (resp *gpb.Empty, err error) {
+	containerPath := filepath.Join("/tmp/libcontainer", a.sandbox.id)
+	factory, err := libcontainer.New(containerPath, libcontainer.Cgroupfs)
+	if err != nil {
+		return emptyResp, err
+	}
+
+	ctr.container, err = factory.Create(req.ContainerId, config)
+	if err != nil {
+		return emptyResp, err
+	}
+	ctr.config = *config
+
+	ctr.initProcess, err = buildProcess(req.OCI.Process, req.ExecId)
+	if err != nil {
+		return emptyResp, err
+	}
+
+	if err = a.execProcess(ctr, ctr.initProcess, true); err != nil {
+		return emptyResp, err
+	}
+
+	if err := a.updateSharedPidNs(ctr); err != nil {
+		return emptyResp, err
+	}
+
+	return emptyResp, a.postExecProcess(ctr, ctr.initProcess)
+}
+
 func (a *agentGRPC) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (resp *gpb.Empty, err error) {
 	if err := a.createContainerChecks(req); err != nil {
 		return emptyResp, err
@@ -578,6 +607,24 @@ func (a *agentGRPC) CreateContainer(ctx context.Context, req *pb.CreateContainer
 		return emptyResp, err
 	}
 
+	if a.sandbox.guestHooksPresent {
+		// Add any custom OCI hooks to the spec
+		a.sandbox.addGuestHooks(ociSpec)
+
+		// Change cwd because libcontainer sets the bundle path to cwd
+		oldcwd, err := pb.ChangeToBundlePath(ociSpec)
+		if err != nil {
+			return emptyResp, err
+		}
+		defer os.Chdir(oldcwd)
+
+		// write the OCI spec to a file so that any hooks can find it
+		err = pb.WriteSpecToFile(ociSpec)
+		if err != nil {
+			return emptyResp, err
+		}
+	}
+
 	// Convert the OCI specification into a libcontainer configuration.
 	config, err := specconv.CreateLibcontainerConfig(&specconv.CreateOpts{
 		CgroupName:   req.ContainerId,
@@ -595,32 +642,7 @@ func (a *agentGRPC) CreateContainer(ctx context.Context, req *pb.CreateContainer
 		return emptyResp, err
 	}
 
-	containerPath := filepath.Join("/tmp/libcontainer", a.sandbox.id)
-	factory, err := libcontainer.New(containerPath, libcontainer.Cgroupfs)
-	if err != nil {
-		return emptyResp, err
-	}
-
-	ctr.container, err = factory.Create(req.ContainerId, config)
-	if err != nil {
-		return emptyResp, err
-	}
-	ctr.config = *config
-
-	ctr.initProcess, err = buildProcess(req.OCI.Process, req.ExecId)
-	if err != nil {
-		return emptyResp, err
-	}
-
-	if err = a.execProcess(ctr, ctr.initProcess, true); err != nil {
-		return emptyResp, err
-	}
-
-	if err := a.updateSharedPidNs(ctr); err != nil {
-		return emptyResp, err
-	}
-
-	return emptyResp, a.postExecProcess(ctr, ctr.initProcess)
+	return a.finishCreateContainer(ctr, req, config)
 }
 
 func (a *agentGRPC) createContainerChecks(req *pb.CreateContainerRequest) (err error) {
@@ -1180,6 +1202,12 @@ func (a *agentGRPC) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequ
 	a.sandbox.running = true
 	a.sandbox.sandboxPidNs = req.SandboxPidns
 	a.sandbox.storages = make(map[string]*sandboxStorage)
+	a.sandbox.guestHooks = &specs.Hooks{}
+	a.sandbox.guestHooksPresent = false
+
+	if req.GuestHookPath != "" {
+		a.sandbox.scanGuestHooks(req.GuestHookPath)
+	}
 
 	if req.SandboxId != "" {
 		a.sandbox.id = req.SandboxId

grpc_test.go

@@ -16,13 +16,14 @@ import (
 	"testing"
 	"time"
 
+	"strconv"
+	"sync"
+
 	pb "github.com/kata-containers/agent/protocols/grpc"
 	"github.com/opencontainers/runc/libcontainer"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
-	"strconv"
-	"sync"
 )
 
 var testSharedPidNs = "testSharedPidNs"