Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
agent: add support for guest-hooks
Browse files Browse the repository at this point in the history
Adds support for running OCI hooks within the guest. A 'drop-in'
path (guest_hook_path) is specified in the cli configuration file
and if set, the agent will look for OCI hooks in this directory
and inject them into the container life cycle.

Fixes: #348
Replaces: #365

Co-authored-by: Edward Guzman <eguzman@nvidia.com>
Co-authored-by: Felix Abecassis <fabecassis@nvidia.com>
Signed-off-by: Edward Guzman <eguzman@nvidia.com>
Signed-off-by: Felix Abecassis <fabecassis@nvidia.com>
  • Loading branch information
flx42 and eguzman3 committed Oct 3, 2018
1 parent 861b18d commit 980023e
Show file tree
Hide file tree
Showing 7 changed files with 578 additions and 206 deletions.
74 changes: 55 additions & 19 deletions agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/opencontainers/runc/libcontainer"
"github.com/opencontainers/runc/libcontainer/configs"
_ "github.com/opencontainers/runc/libcontainer/nsenter"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"golang.org/x/sys/unix"
Expand Down Expand Up @@ -98,25 +99,27 @@ type sandboxStorage struct {
type sandbox struct {
sync.RWMutex

id string
hostname string
containers map[string]*container
channel channel
network network
wg sync.WaitGroup
sharedPidNs namespace
mounts []string
subreaper reaper
server *grpc.Server
pciDeviceMap map[string]string
deviceWatchers map[string](chan string)
sharedUTSNs namespace
sharedIPCNs namespace
running bool
noPivotRoot bool
enableGrpcTrace bool
sandboxPidNs bool
storages map[string]*sandboxStorage
id string
hostname string
containers map[string]*container
channel channel
network network
wg sync.WaitGroup
sharedPidNs namespace
mounts []string
subreaper reaper
server *grpc.Server
pciDeviceMap map[string]string
deviceWatchers map[string](chan string)
sharedUTSNs namespace
sharedIPCNs namespace
guestHooks *specs.Hooks
guestHooksPresent bool
running bool
noPivotRoot bool
enableGrpcTrace bool
sandboxPidNs bool
storages map[string]*sandboxStorage
}

var agentFields = logrus.Fields{
Expand Down Expand Up @@ -240,6 +243,39 @@ func (s *sandbox) setSandboxStorage(path string) bool {
return false
}

// scanGuestHooks will search the given guestHookPath
// for any OCI hooks
func (s *sandbox) scanGuestHooks(guestHookPath string) {
fieldLogger := agentLog.WithField("oci-hook-path", guestHookPath)
fieldLogger.Info("Scanning guest filesystem for OCI hooks")

s.guestHooks.Prestart = findHooks(guestHookPath, "prestart")
s.guestHooks.Poststart = findHooks(guestHookPath, "poststart")
s.guestHooks.Poststop = findHooks(guestHookPath, "poststop")

if len(s.guestHooks.Prestart) > 0 || len(s.guestHooks.Poststart) > 0 || len(s.guestHooks.Poststop) > 0 {
s.guestHooksPresent = true
} else {
fieldLogger.Warn("Guest hooks were requested but none were found")
}
}

// addGuestHooks will add any guest OCI hooks that were
// found to the OCI spec
func (s *sandbox) addGuestHooks(spec *specs.Spec) {
if spec == nil {
return
}

if spec.Hooks == nil {
spec.Hooks = &specs.Hooks{}
}

spec.Hooks.Prestart = append(spec.Hooks.Prestart, s.guestHooks.Prestart...)
spec.Hooks.Poststart = append(spec.Hooks.Poststart, s.guestHooks.Poststart...)
spec.Hooks.Poststop = append(spec.Hooks.Poststop, s.guestHooks.Poststop...)
}

// unSetSandboxStorage will decrement the sandbox storage
// reference counter. If there aren't any containers using
// that sandbox storage, this method will remove the
Expand Down
52 changes: 52 additions & 0 deletions agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,18 @@ package main
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"reflect"
"strings"
"syscall"
"testing"

"google.golang.org/grpc"

pb "github.com/kata-containers/agent/protocols/grpc"
"github.com/opencontainers/runc/libcontainer"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
Expand Down Expand Up @@ -446,3 +449,52 @@ func TestGetCgroupMountsSuccessful(t *testing.T) {
err = syscall.Unmount(cgprocDir, 0)
assert.Nil(t, err, "%v", err)
}

func TestAddGuestHooks(t *testing.T) {
assert := assert.New(t)

hookPath, err := ioutil.TempDir("", "hooks")
assert.NoError(err)
defer os.RemoveAll(hookPath)

poststopPath := path.Join(hookPath, "poststop")
err = os.Mkdir(poststopPath, 0750)
assert.NoError(err)

dirPath := path.Join(poststopPath, "directory")
err = os.Mkdir(dirPath, 0750)
assert.NoError(err)

normalPath := path.Join(poststopPath, "normalfile")
f, err := os.OpenFile(normalPath, os.O_RDONLY|os.O_CREATE, 0640)
assert.NoError(err)
f.Close()

symlinkPath := path.Join(poststopPath, "symlink")
err = os.Link(normalPath, symlinkPath)
assert.NoError(err)

s := &sandbox{
guestHooks: &specs.Hooks{},
guestHooksPresent: false,
}

s.scanGuestHooks(hookPath)
assert.False(s.guestHooksPresent)

spec := &specs.Spec{}
s.addGuestHooks(spec)
assert.True(len(spec.Hooks.Poststop) == 0)

execPath := path.Join(poststopPath, "executable")
f, err = os.OpenFile(execPath, os.O_RDONLY|os.O_CREATE, 0750)
assert.NoError(err)
f.Close()

s.scanGuestHooks(hookPath)
assert.True(s.guestHooksPresent)

s.addGuestHooks(spec)
assert.True(len(spec.Hooks.Poststop) == 1)
assert.True(strings.Contains(spec.Hooks.Poststop[0].Path, "executable"))
}
81 changes: 55 additions & 26 deletions grpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,35 @@ func (a *agentGRPC) rollbackFailingContainerCreation(ctr *container) {
}
}

func (a *agentGRPC) finishCreateContainer(ctr *container, req *pb.CreateContainerRequest, config *configs.Config) (resp *gpb.Empty, err error) {
containerPath := filepath.Join("/tmp/libcontainer", a.sandbox.id)
factory, err := libcontainer.New(containerPath, libcontainer.Cgroupfs)
if err != nil {
return emptyResp, err
}

ctr.container, err = factory.Create(req.ContainerId, config)
if err != nil {
return emptyResp, err
}
ctr.config = *config

ctr.initProcess, err = buildProcess(req.OCI.Process, req.ExecId)
if err != nil {
return emptyResp, err
}

if err = a.execProcess(ctr, ctr.initProcess, true); err != nil {
return emptyResp, err
}

if err := a.updateSharedPidNs(ctr); err != nil {
return emptyResp, err
}

return emptyResp, a.postExecProcess(ctr, ctr.initProcess)
}

func (a *agentGRPC) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (resp *gpb.Empty, err error) {
if err := a.createContainerChecks(req); err != nil {
return emptyResp, err
Expand Down Expand Up @@ -591,6 +620,25 @@ func (a *agentGRPC) CreateContainer(ctx context.Context, req *pb.CreateContainer
return emptyResp, err
}

if a.sandbox.guestHooksPresent {
// Add any custom OCI hooks to the spec
a.sandbox.addGuestHooks(ociSpec)

// write the OCI spec to a file so that hooks can read it
err = writeSpecToFile(ociSpec)
if err != nil {
return emptyResp, err
}

// Change cwd because libcontainer assumes the bundle path is the cwd:
// https://github.com/opencontainers/runc/blob/v1.0.0-rc5/libcontainer/specconv/spec_linux.go#L157
oldcwd, err := changeToBundlePath(ociSpec)
if err != nil {
return emptyResp, err
}
defer os.Chdir(oldcwd)
}

// Convert the OCI specification into a libcontainer configuration.
config, err := specconv.CreateLibcontainerConfig(&specconv.CreateOpts{
CgroupName: req.ContainerId,
Expand All @@ -608,32 +656,7 @@ func (a *agentGRPC) CreateContainer(ctx context.Context, req *pb.CreateContainer
return emptyResp, err
}

containerPath := filepath.Join("/tmp/libcontainer", a.sandbox.id)
factory, err := libcontainer.New(containerPath, libcontainer.Cgroupfs)
if err != nil {
return emptyResp, err
}

ctr.container, err = factory.Create(req.ContainerId, config)
if err != nil {
return emptyResp, err
}
ctr.config = *config

ctr.initProcess, err = buildProcess(req.OCI.Process, req.ExecId)
if err != nil {
return emptyResp, err
}

if err = a.execProcess(ctr, ctr.initProcess, true); err != nil {
return emptyResp, err
}

if err := a.updateSharedPidNs(ctr); err != nil {
return emptyResp, err
}

return emptyResp, a.postExecProcess(ctr, ctr.initProcess)
return a.finishCreateContainer(ctr, req, config)
}

func (a *agentGRPC) createContainerChecks(req *pb.CreateContainerRequest) (err error) {
Expand Down Expand Up @@ -1193,6 +1216,12 @@ func (a *agentGRPC) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequ
a.sandbox.running = true
a.sandbox.sandboxPidNs = req.SandboxPidns
a.sandbox.storages = make(map[string]*sandboxStorage)
a.sandbox.guestHooks = &specs.Hooks{}
a.sandbox.guestHooksPresent = false

if req.GuestHookPath != "" {
a.sandbox.scanGuestHooks(req.GuestHookPath)
}

if req.SandboxId != "" {
a.sandbox.id = req.SandboxId
Expand Down
112 changes: 112 additions & 0 deletions oci.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
//
// Copyright (c) 2018 NVIDIA CORPORATION
//
// SPDX-License-Identifier: Apache-2.0
//

package main

import (
"encoding/json"
"errors"
"io/ioutil"
"os"
"path"
"path/filepath"

"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)

// OCI config file
const (
ociConfigFile string = "config.json"
ociConfigFileMode os.FileMode = 0444
)

// writeSpecToFile writes the container's OCI spec to "dirname(spec.Root.Path)/config.json"
// This effectively makes the parent directory a valid OCI bundle.
func writeSpecToFile(spec *specs.Spec) error {
bundlePath := filepath.Dir(spec.Root.Path)
configPath := filepath.Join(bundlePath, ociConfigFile)
f, err := os.OpenFile(configPath, os.O_WRONLY|os.O_CREATE, ociConfigFileMode)
if err != nil {
return err
}
defer f.Close()

return json.NewEncoder(f).Encode(spec)
}

// changeToBundlePath changes the cwd to the OCI bundle path defined as
// dirname(spec.Root.Path) and returns the old cwd.
func changeToBundlePath(spec *specs.Spec) (string, error) {
cwd, err := os.Getwd()
if err != nil {
return cwd, err
}

if spec == nil || spec.Root == nil || spec.Root.Path == "" {
return cwd, errors.New("invalid OCI spec")
}

bundlePath := filepath.Dir(spec.Root.Path)
configPath := filepath.Join(bundlePath, ociConfigFile)

// Verify that config.json is present at the root of the bundle path.
if _, err := os.Stat(configPath); err != nil {
return cwd, errors.New("invalid OCI bundle")
}

return cwd, os.Chdir(bundlePath)
}

func isValidHook(file os.FileInfo) (bool, error) {
if file.IsDir() {
return false, errors.New("is a directory")
}

mode := file.Mode()
if (mode & os.ModeSymlink) != 0 {
return false, errors.New("is a symbolic link")
}

perm := mode & os.ModePerm
if (perm & 0111) == 0 {
return false, errors.New("is not executable")
}

return true, nil
}

// findHooks searches guestHookPath for any OCI hooks for a given hookType
func findHooks(guestHookPath, hookType string) (hooksFound []specs.Hook) {
hooksPath := path.Join(guestHookPath, hookType)

files, err := ioutil.ReadDir(hooksPath)
if err != nil {
agentLog.WithError(err).WithField("oci-hook-type", hookType).Info("Skipping hook type")
return
}

for _, file := range files {
name := file.Name()
if ok, err := isValidHook(file); !ok {
agentLog.WithError(err).WithField("oci-hook-name", name).Warn("Skipping hook")
continue
}

agentLog.WithFields(logrus.Fields{
"oci-hook-name": name,
"oci-hook-type": hookType,
}).Info("Adding hook")
hooksFound = append(hooksFound, specs.Hook{
Path: path.Join(hooksPath, name),
Args: []string{name, hookType},
})
}

agentLog.WithField("oci-hook-type", hookType).Infof("Added %d hooks", len(hooksFound))

return
}
Loading

0 comments on commit 980023e

Please sign in to comment.