Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
vitiofsd: Add virtiofsd interaface
Browse files Browse the repository at this point in the history 
In oderder to make unit testing simpler,
lets add an interface that could be mocked.

Let hypervisor have a instance of virtiofsd interface,
and this makes a loose dependency to allow mock testing.

With the inteface is possible to add startSandbox unit test:

- use utils.StartCmd to mock call to start hypervisor process.

- Add unit test for startSandbox.

Fixes: #2367

Signed-off-by: Jose Carlos Venegas Munoz <jose.carlos.venegas.munoz@intel.com>
  • Loading branch information
@jcvenegas
jcvenegas committed Dec 20, 2019
1 parent 2a085ee commit a2d3f9f
Show file tree
Hide file tree
Showing 6 changed files with 439 additions and 179 deletions.
257 changes: 78 additions & 179 deletions virtcontainers/clh.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
package virtcontainers

import (
"bufio"
"bytes"
"context"
"encoding/json"
Expand Down Expand Up @@ -52,7 +51,6 @@ const (
const (
// Values are mandatory by http API
// Values based on:
// github.com/cloud-hypervisor/cloud-hypervisor/blob/v0.3.0/vmm/src/config.rs#L395
clhTimeout = 10
clhAPITimeout = 1
clhStopSandboxTimeout = 3
Expand Down Expand Up @@ -113,6 +111,7 @@ type cloudHypervisor struct {
version CloudHypervisorVersion
vmconfig chclient.VmConfig
cmdOutput bytes.Buffer
virtiofsd Virtiofsd
}

var clhKernelParams = []Param{
Expand Down Expand Up @@ -182,10 +181,26 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ

clh.Logger().WithField("function", "createSandbox").Info("creating Sandbox")

virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id)
if err != nil {
return nil

}

// No need to return an error from there since there might be nothing
// to fetch if this is the first time the hypervisor is created.
if err := clh.store.Load(store.Hypervisor, &clh.state); err != nil {
clh.Logger().WithField("function", "createSandbox").WithError(err).Info("No info could be fetched")
err = clh.store.Load(store.Hypervisor, &clh.state)
if err != nil {
clh.Logger().WithField("function", "createSandbox").WithError(err).Info("Sandbox not found creating ")
} else {
clh.Logger().WithField("function", "createSandbox").Info("Sandbox already exist, loading from state")
clh.virtiofsd = &virtiofsd{
PID: clh.state.VirtiofsdPID,
sourcePath: filepath.Join(kataHostSharedDir(), clh.id),
debug: clh.config.Debug,
socketPath: virtiofsdSocketPath,
}
return nil
}

// Set initial memomory size of the virtual machine
Expand Down Expand Up @@ -269,6 +284,15 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ
}
clh.state.apiSocket = apiSocketPath

clh.virtiofsd = &virtiofsd{
path: clh.config.VirtioFSDaemon,
sourcePath: filepath.Join(kataHostSharedDir(), clh.id),
socketPath: virtiofsdSocketPath,
extraArgs: clh.config.VirtioFSExtraArgs,
debug: clh.config.Debug,
cache: clh.config.VirtioFSCache,
}

return nil
}

Expand All @@ -288,12 +312,17 @@ func (clh *cloudHypervisor) startSandbox(timeout int) error {
return err
}

if clh.virtiofsd == nil {
return errors.New("Missing virtiofsd configuration")
}

if clh.config.SharedFS == config.VirtioFS {
clh.Logger().WithField("function", "startSandbox").Info("Starting virtiofsd")
_, err = clh.setupVirtiofsd(timeout)
pid, err := clh.virtiofsd.Start(ctx)
if err != nil {
return err
}
clh.state.VirtiofsdPID = pid
if err = clh.storeState(); err != nil {
return err
}
Expand All @@ -310,7 +339,7 @@ func (clh *cloudHypervisor) startSandbox(timeout int) error {

if err := clh.waitVMM(clhTimeout); err != nil {
clh.Logger().WithField("error", err).WithField("output", clh.cmdOutput.String()).Warn("cloud-hypervisor init failed")
if shutdownErr := clh.shutdownVirtiofsd(); shutdownErr != nil {
if shutdownErr := clh.virtiofsd.Stop(); shutdownErr != nil {
clh.Logger().WithField("error", shutdownErr).Warn("error shutting down Virtiofsd")
}
return err
Expand Down Expand Up @@ -501,64 +530,36 @@ func (clh *cloudHypervisor) terminate() (err error) {
span, _ := clh.trace("terminate")
defer span.Finish()

defer func() {
if err != nil {
clh.Logger().Info("Terminate Cloud Hypervisor failed")
} else {
clh.Logger().Info("Cloud Hypervisor stopped")
clh.reset()
clh.Logger().Debug("removing virtiofsd and vm sockets")
path, err := clh.virtioFsSocketPath(clh.id)
if err == nil {
rerr := os.Remove(path)
if rerr != nil {
clh.Logger().WithField("path", path).Warn("removing virtiofsd socket failed")
}
}
path, err = clh.vsockSocketPath(clh.id)
if err == nil {
rerr := os.Remove(path)
if rerr != nil {
clh.Logger().WithField("path", path).Warn("removing vm socket failed")
}
}
}

_ = clh.cleanupVM(true)
}()

pid := clh.state.PID
pidRunning := true
if pid == 0 {
clh.Logger().WithField("PID", pid).Info("Skipping kill cloud hypervisor. invalid pid")
return nil
pidRunning = false
}
clh.Logger().WithField("PID", pid).Info("Stopping Cloud Hypervisor")

clhRunning, err := clh.isClhRunning(clhStopSandboxTimeout)

if err != nil {
return err
}

if !clhRunning {
return nil
}

ctx, cancel := context.WithTimeout(context.Background(), clhStopSandboxTimeout*time.Second)
defer cancel()
clh.Logger().WithField("PID", pid).Info("Stopping Cloud Hypervisor")

if _, err = clh.client().ShutdownVMM(ctx); err != nil {
return err
if pidRunning {
clhRunning, _ := clh.isClhRunning(clhStopSandboxTimeout)
if clhRunning {
ctx, cancel := context.WithTimeout(context.Background(), clhStopSandboxTimeout*time.Second)
defer cancel()
if _, err = clh.client().ShutdownVMM(ctx); err != nil {
return err
}
}
}

// At this point the VMM was stop nicely, but need to check if PID is still running
// Wait for the VM process to terminate
tInit := time.Now()
for {
if err = syscall.Kill(pid, syscall.Signal(0)); err != nil {
return nil
pidRunning = false
break
}

if time.Since(tInit).Seconds() >= clhStopSandboxTimeout {
pidRunning = true
clh.Logger().Warnf("VM still running after waiting %ds", clhStopSandboxTimeout)
break
}
Expand All @@ -569,7 +570,21 @@ func (clh *cloudHypervisor) terminate() (err error) {

// Let's try with a hammer now, a SIGKILL should get rid of the
// VM process.
return syscall.Kill(pid, syscall.SIGKILL)
if pidRunning {
if err = syscall.Kill(pid, syscall.SIGKILL); err != nil {
return fmt.Errorf("Fatal, failed to kill hypervisor process, error: %s", err)
}
}

if clh.virtiofsd == nil {
return errors.New("virtiofsd config is nil, failed to stop it")
}

if err := clh.cleanupVM(true); err != nil {
return err
}

return clh.virtiofsd.Stop()
}

func (clh *cloudHypervisor) reset() {
Expand Down Expand Up @@ -598,133 +613,6 @@ func (clh *cloudHypervisor) generateSocket(id string, useVsock bool) (interface{
}, nil
}

func (clh *cloudHypervisor) setupVirtiofsd(timeout int) (remain int, err error) {

if clh.config.VirtioFSDaemon == "" {
return timeout, errors.New("Virtiofsd path is empty")
}

sockPath, perr := clh.virtioFsSocketPath(clh.id)
if perr != nil {
return 0, perr
}

theArgs, err := clh.virtiofsdArgs(sockPath)
if err != nil {
return 0, err
}

clh.Logger().WithField("path", clh.config.VirtioFSDaemon).Info()
clh.Logger().WithField("args", strings.Join(theArgs, " ")).Info()

cmd := exec.Command(clh.config.VirtioFSDaemon, theArgs...)
stderr, err := cmd.StderrPipe()
if err != nil {
return 0, err
}

if err = cmd.Start(); err != nil {
return 0, err
}
defer func() {
if err != nil {
clh.state.VirtiofsdPID = 0
cmd.Process.Kill()
} else {
clh.state.VirtiofsdPID = cmd.Process.Pid

}
clh.storeState()
}()

// Wait for socket to become available
sockReady := make(chan error, 1)
timeStart := time.Now()
go func() {
scanner := bufio.NewScanner(stderr)
var sent bool
for scanner.Scan() {
if clh.config.Debug {
clh.Logger().WithField("source", "virtiofsd").Debug(scanner.Text())
}
if !sent && strings.Contains(scanner.Text(), "Waiting for vhost-user socket connection...") {
sockReady <- nil
sent = true
}
}
if !sent {
if err := scanner.Err(); err != nil {
sockReady <- err
} else {
sockReady <- fmt.Errorf("virtiofsd did not announce socket connection")
}
}
clh.Logger().Info("virtiofsd quits")
// Wait to release resources of virtiofsd process
cmd.Process.Wait()

}()

return clh.waitVirtiofsd(timeStart, timeout, sockReady,
fmt.Sprintf("virtiofsd (pid=%d) socket %s", cmd.Process.Pid, sockPath))
}

func (clh *cloudHypervisor) waitVirtiofsd(start time.Time, timeout int, ready chan error, errMsg string) (int, error) {
var err error

timeoutDuration := time.Duration(timeout) * time.Second
select {
case err = <-ready:
case <-time.After(timeoutDuration):
err = fmt.Errorf("timed out waiting for %s", errMsg)
}
if err != nil {
return 0, err
}

// Now reduce timeout by the elapsed time
elapsed := time.Since(start)
if elapsed < timeoutDuration {
timeout = timeout - int(elapsed.Seconds())
} else {
timeout = 0
}
return timeout, nil
}

func (clh *cloudHypervisor) virtiofsdArgs(sockPath string) ([]string, error) {

sourcePath := filepath.Join(kataHostSharedDir(), clh.id)
if _, err := os.Stat(sourcePath); os.IsNotExist(err) {
if err = os.MkdirAll(sourcePath, os.ModePerm); err != nil {
return nil, err
}
}

args := []string{
"-f",
"-o", "vhost_user_socket=" + sockPath,
"-o", "source=" + sourcePath,
"-o", "cache=" + clh.config.VirtioFSCache}

if len(clh.config.VirtioFSExtraArgs) != 0 {
args = append(args, clh.config.VirtioFSExtraArgs...)
}
return args, nil
}

func (clh *cloudHypervisor) shutdownVirtiofsd() (err error) {

err = syscall.Kill(clh.state.VirtiofsdPID, syscall.SIGKILL)

if err != nil {
clh.state.VirtiofsdPID = 0
clh.storeState()
}
return err

}

func (clh *cloudHypervisor) virtioFsSocketPath(id string) (string, error) {
return utils.BuildSocketPath(store.RunVMStoragePath(), id, virtioFsSocket)
}
Expand Down Expand Up @@ -868,7 +756,7 @@ func (clh *cloudHypervisor) LaunchClh() (string, int, error) {
cmd.Env = append(cmd.Env, "RUST_BACKTRACE=full")
}

if err := cmd.Start(); err != nil {
if err := utils.StartCmd(cmd); err != nil {
fmt.Println("Error starting cloudHypervisor", err)
if cmd.Process != nil {
cmd.Process.Kill()
Expand Down Expand Up @@ -1120,6 +1008,15 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error {
return errors.New("Hypervisor ID is empty")
}

clh.Logger().Debug("removing vm sockets")

path, err := clh.vsockSocketPath(clh.id)
if err == nil {
if err := os.Remove(path); err != nil {
clh.Logger().WithField("path", path).Warn("removing vm socket failed")
}
}

// cleanup vm path
dir := filepath.Join(store.RunVMStoragePath(), clh.id)

Expand Down Expand Up @@ -1166,5 +1063,7 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error {
}
}

clh.reset()

return nil
}
Loading

0 comments on commit a2d3f9f

Please sign in to comment.