Merge pull request #2202 from lifupan/watch_firecracker_console

FC: log out the firecracker's console when debug enabled

Gopkg.toml

@@ -78,6 +78,10 @@
   branch = "master"
   name = "github.com/hashicorp/yamux"
 
+[[constraint]]
+  revision = "0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f"
+  name = "github.com/containerd/console"
+
 [prune]
   non-go = true
   go-tests = true

virtcontainers/acrn.go

@@ -348,7 +348,7 @@ func (a *Acrn) createDummyVirtioBlkDev(devices []Device) ([]Device, error) {
 }
 
 // createSandbox is the Hypervisor sandbox creation.
-func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
+func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error {
 	// Save the tracing context
 	a.ctx = ctx
 

virtcontainers/acrn_test.go

@@ -230,7 +230,7 @@ func TestAcrnCreateSandbox(t *testing.T) {
 	//set PID to 1 to ignore hypercall to get UUID and set a random UUID
 	a.state.PID = 1
 	a.state.UUID = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6"
-	err = a.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil)
+	err = a.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false)
 	assert.NoError(err)
 	assert.Exactly(acrnConfig, a.config)
 }

virtcontainers/fc.go

@@ -6,8 +6,10 @@
 package virtcontainers
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -31,6 +33,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/blang/semver"
+	"github.com/containerd/console"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	fcmodels "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models"
 	"github.com/kata-containers/runtime/virtcontainers/store"
@@ -77,16 +80,12 @@ var fcKernelParams = append(commonVirtioblkKernelRootParams, []Param{
 	{"reboot", "k"},
 	{"panic", "1"},
 	{"iommu", "off"},
-	{"8250.nr_uarts", "0"},
 	{"net.ifnames", "0"},
 	{"random.trust_cpu", "on"},
 
 	// Firecracker doesn't support ACPI
 	// Fix kernel error "ACPI BIOS Error (bug)"
 	{"acpi", "off"},
-
-	// Tell agent where to send the logs
-	{"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)},
 }...)
 
 func (s vmmState) String() string {
@@ -141,8 +140,9 @@ type firecracker struct {
 	config         HypervisorConfig
 	pendingDevices []firecrackerDevice // Devices to be added when the FC API is ready
 
-	state  firecrackerState
-	jailed bool //Set to true if jailer is enabled
+	state    firecrackerState
+	jailed   bool //Set to true if jailer is enabled
+	stateful bool //Set to true if running with shimv2
 }
 
 type firecrackerDevice struct {
@@ -211,7 +211,7 @@ func (fc *firecracker) bindMount(ctx context.Context, source, destination string
 
 // For firecracker this call only sets the internal structure up.
 // The sandbox will be created and started through startSandbox().
-func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error {
+func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore, stateful bool) error {
 	fc.ctx = ctx
 
 	span, _ := fc.trace("createSandbox")
@@ -223,6 +223,7 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N
 	fc.store = vcStore
 	fc.state.set(notReady)
 	fc.config = *hypervisorConfig
+	fc.stateful = stateful
 
 	// When running with jailer all resources need to be under
 	// a specific location and that location needs to have
@@ -386,22 +387,25 @@ func (fc *firecracker) fcInit(timeout int) error {
 	var args []string
 	var cmd *exec.Cmd
 
+	if !fc.config.Debug && fc.stateful {
+		args = append(args, "--daemonize")
+	}
+
 	//https://github.com/firecracker-microvm/firecracker/blob/master/docs/jailer.md#jailer-usage
 	//--seccomp-level specifies whether seccomp filters should be installed and how restrictive they should be. Possible values are:
 	//0 : disabled.
 	//1 : basic filtering. This prohibits syscalls not whitelisted by Firecracker.
 	//2 (default): advanced filtering. This adds further checks on some of the parameters of the allowed syscalls.
 	if fc.jailed {
-		args = []string{
+		args = append(args,
 			"--id", fc.id,
 			"--node", "0", //FIXME: Comprehend NUMA topology or explicit ignore
 			"--seccomp-level", "2",
 			"--exec-file", fc.config.HypervisorPath,
 			"--uid", "0", //https://github.com/kata-containers/runtime/issues/1869
 			"--gid", "0",
 			"--chroot-base-dir", fc.chrootBaseDir,
-			"--daemonize",
-		}
+		)
 		if fc.netNSPath != "" {
 			args = append(args, "--netns", fc.netNSPath)
 		}
@@ -412,6 +416,16 @@ func (fc *firecracker) fcInit(timeout int) error {
 
 	}
 
+	if fc.config.Debug && fc.stateful {
+		stdin, err := fc.watchConsole()
+		if err != nil {
+			return err
+		}
+
+		cmd.Stderr = stdin
+		cmd.Stdout = stdin
+	}
+
 	fc.Logger().WithField("hypervisor args", args).Debug()
 	fc.Logger().WithField("hypervisor cmd", cmd).Debug()
 	if err := cmd.Start(); err != nil {
@@ -660,6 +674,16 @@ func (fc *firecracker) startSandbox(timeout int) error {
 		return err
 	}
 
+	if fc.config.Debug && fc.stateful {
+		fcKernelParams = append(fcKernelParams, Param{"console", "ttyS0"})
+	} else {
+		fcKernelParams = append(fcKernelParams, []Param{
+			{"8250.nr_uarts", "0"},
+			// Tell agent where to send the logs
+			{"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)},
+		}...)
+	}
+
 	kernelParams := append(fc.config.KernelParams, fcKernelParams...)
 	strParams := SerializeParams(kernelParams, "=")
 	formattedParams := strings.Join(strParams, " ")
@@ -1097,3 +1121,37 @@ func (fc *firecracker) generateSocket(id string, useVsock bool) (interface{}, er
 		Port:    uint32(vSockPort),
 	}, nil
 }
+
+func (fc *firecracker) watchConsole() (*os.File, error) {
+	master, slave, err := console.NewPty()
+	if err != nil {
+		fc.Logger().WithField("Error create pseudo tty", err).Debug()
+		return nil, err
+	}
+
+	stdio, err := os.OpenFile(slave, syscall.O_RDWR, 0700)
+	if err != nil {
+		fc.Logger().WithError(err).Debugf("open pseudo tty %s", slave)
+		return nil, err
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(master)
+		for scanner.Scan() {
+			fc.Logger().WithFields(logrus.Fields{
+				"sandbox":   fc.id,
+				"vmconsole": scanner.Text(),
+			}).Infof("reading guest console")
+		}
+
+		if err := scanner.Err(); err != nil {
+			if err == io.EOF {
+				fc.Logger().Info("console watcher quits")
+			} else {
+				fc.Logger().WithError(err).Error("Failed to read guest console")
+			}
+		}
+	}()
+
+	return stdio, nil
+}

virtcontainers/hypervisor.go

@@ -726,7 +726,7 @@ func generateVMSocket(id string, useVsock bool) (interface{}, error) {
 // hypervisor is the virtcontainers hypervisor interface.
 // The default hypervisor implementation is Qemu.
 type hypervisor interface {
-	createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error
+	createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error
 	startSandbox(timeout int) error
 	stopSandbox() error
 	pauseSandbox() error

virtcontainers/mock_hypervisor.go

@@ -27,7 +27,7 @@ func (m *mockHypervisor) hypervisorConfig() HypervisorConfig {
 	return HypervisorConfig{}
 }
 
-func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
+func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error {
 	err := hypervisorConfig.valid()
 	if err != nil {
 		return err

virtcontainers/mock_hypervisor_test.go

@@ -31,7 +31,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) {
 	ctx := context.Background()
 
 	// wrong config
-	err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil)
+	err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false)
 	assert.Error(err)
 
 	sandbox.config.HypervisorConfig = HypervisorConfig{
@@ -40,7 +40,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) {
 		HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor),
 	}
 
-	err = m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil)
+	err = m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false)
 	assert.NoError(err)
 }
 

virtcontainers/qemu.go

@@ -463,7 +463,7 @@ func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memo
 }
 
 // createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
-func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error {
+func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore, stateful bool) error {
 	// Save the tracing context
 	q.ctx = ctx
 

virtcontainers/qemu_test.go

@@ -99,7 +99,7 @@ func TestQemuCreateSandbox(t *testing.T) {
 	parentDir := store.SandboxConfigurationRootPath(sandbox.id)
 	assert.NoError(os.MkdirAll(parentDir, store.DirMode))
 
-	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store)
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false)
 	assert.NoError(err)
 	assert.NoError(os.RemoveAll(parentDir))
 	assert.Exactly(qemuConfig, q.config)
@@ -131,7 +131,7 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) {
 	parentDir := store.SandboxConfigurationRootPath(sandbox.id)
 	assert.NoError(os.RemoveAll(parentDir))
 
-	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store)
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false)
 	assert.NoError(err)
 }
 
@@ -429,7 +429,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 
 	q := &qemu{}
 	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
-	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store)
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false)
 	assert.NoError(err)
 
 	assert.Equal(q.qemuConfig.Knobs.FileBackedMem, true)
@@ -445,7 +445,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
 	sandbox.config.HypervisorConfig.MemoryPath = fallbackFileBackedMemDir
 
-	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store)
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false)
 
 	expectErr := errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work")
 	assert.Equal(expectErr.Error(), err.Error())
@@ -456,7 +456,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 
 	q = &qemu{}
 	sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc"
-	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store)
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false)
 	assert.NoError(err)
 	assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false)
 	assert.Equal(q.qemuConfig.Knobs.MemShared, false)

virtcontainers/sandbox.go

@@ -569,7 +569,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		s.Restore()
 
 		// new store doesn't require hypervisor to be stored immediately
-		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil); err != nil {
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil, s.stateful); err != nil {
 			return nil, err
 		}
 	} else {
@@ -591,7 +591,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 			s.state = state
 		}
 
-		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil {
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store, s.stateful); err != nil {
 			return nil, err
 		}
 	}

virtcontainers/vm.go

@@ -172,7 +172,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
 		}
 	}()
 
-	if err = hypervisor.createSandbox(ctx, id, NetworkNamespace{}, &config.HypervisorConfig, vcStore); err != nil {
+	if err = hypervisor.createSandbox(ctx, id, NetworkNamespace{}, &config.HypervisorConfig, vcStore, false); err != nil {
 		return nil, err
 	}