agent: connect debugging console in a specific vsock port

some hypervisors, like firecracker, don't have connected a unix socket to
`/dev/console` making impossible to start a debugging console.
Use vsock connection as standard input and output (std*) for the shell
process.
Add cmdline option to specify the vsock port to connect the debugging console.

fixes #666

Signed-off-by: Julio Montes <julio.montes@intel.com>

README.md

@@ -37,6 +37,32 @@ allow the agent process to start a debug console. Debug console is only availabl
 or `sh` is installed in the rootfs or initrd image. Developers can [connect to the virtual
 machine using the debug console](https://github.com/kata-containers/documentation/blob/master/Developer-Guide.md#connect-to-the-virtual-machine-using-the-debug-console)
 
+### Enable debug console for firecracker
+
+Firecracker doesn't have a UNIX socket connected to `/dev/console`, hence the
+kernel command line option `agent.debug_console` will not work for firecracker.
+Fortunately, firecracker supports [`hybrid vsocks`][1], and they can be used to
+communicate processes in the guest with processes in the host.
+The kernel command line option `agent.debug_console_vport` was added to allow
+developers specify on which `vsock` port the debugging console should be connected.
+
+In firecracker, the UNIX socket that is connected to the `vsock` end is created at
+`/var/lib/vc/firecracker/$CID/root/kata.hvsock`, where `$CID` is the container ID.
+
+Run the following commands to have a debugging console in firecracker.
+
+```sh
+$ conf="/usr/share/defaults/kata-containers/configuration.toml"
+$ sudo sed -i 's/^kernel_params.*/kernel_params="agent.debug_console_vport=1026"/g' "${conf}"
+$ sudo su -c 'cd /var/lib/vc/firecracker/08facf/root/ && socat stdin unix-connect:kata.hvsock'
+CONNECT 1026
+```
+
+**NOTE:** Ports 1024 and 1025 are reserved for communication with the agent and gathering of agent logs respectively
+
 ## `cpuset` cgroup details
 
 See the [cpuset cgroup documentation](documentation/features/cpuset.md).
+
+
+[1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md

agent.go

@@ -171,6 +171,9 @@ var debugConsole = false
 // Specify a vsock port where logs are written.
 var logsVSockPort = uint32(0)
 
+// Specify a vsock port where debug console is attached.
+var debugConsoleVSockPort = uint32(0)
+
 // commType is used to denote the communication channel type used.
 type commType int
 
@@ -1287,11 +1290,75 @@ func cgroupsMount() error {
 	return ioutil.WriteFile(cgroupMemoryUseHierarchyPath, []byte{'1'}, cgroupMemoryUseHierarchyMode)
 }
 
+func setupDebugConsoleForVsock(ctx context.Context) error {
+	var shellPath string
+	for _, s := range supportedShells {
+		var err error
+		if _, err = os.Stat(s); err == nil {
+			shellPath = s
+			break
+		}
+		agentLog.WithError(err).WithField("shell", s).Warn("Shell not found")
+	}
+
+	if shellPath == "" {
+		return fmt.Errorf("No available shells (checked %v)", supportedShells)
+	}
+
+	cmd := exec.Command(shellPath, "-i")
+	cmd.Env = os.Environ()
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		// Create Session
+		Setsid: true,
+	}
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				// stop the thread
+				return
+			default:
+				dcmd := *cmd
+
+				l, err := vsock.Listen(debugConsoleVSockPort)
+				if err != nil {
+					// nobody dialing
+					continue
+				}
+				c, err := l.Accept()
+				if err != nil {
+					l.Close()
+					// no connection
+					continue
+				}
+
+				dcmd.Stdin = c
+				dcmd.Stdout = c
+				dcmd.Stderr = c
+
+				if err := dcmd.Run(); err != nil {
+					agentLog.WithError(err).Warn("failed to start debug console")
+				}
+
+				c.Close()
+				l.Close()
+			}
+		}
+	}()
+
+	return nil
+}
+
 func setupDebugConsole(ctx context.Context, debugConsolePath string) error {
 	if !debugConsole {
 		return nil
 	}
 
+	if debugConsoleVSockPort != uint32(0) {
+		return setupDebugConsoleForVsock(ctx)
+	}
+
 	var shellPath string
 	for _, s := range supportedShells {
 		var err error

config.go

@@ -17,19 +17,20 @@ import (
 )
 
 const (
-	optionPrefix      = "agent."
-	logLevelFlag      = optionPrefix + "log"
-	logsVSockPortFlag = optionPrefix + "log_vport"
-	devModeFlag       = optionPrefix + "devmode"
-	traceModeFlag     = optionPrefix + "trace"
-	useVsockFlag      = optionPrefix + "use_vsock"
-	debugConsoleFlag  = optionPrefix + "debug_console"
-	kernelCmdlineFile = "/proc/cmdline"
-	traceModeStatic   = "static"
-	traceModeDynamic  = "dynamic"
-	traceTypeIsolated = "isolated"
-	traceTypeCollated = "collated"
-	defaultTraceType  = traceTypeIsolated
+	optionPrefix          = "agent."
+	logLevelFlag          = optionPrefix + "log"
+	logsVSockPortFlag     = optionPrefix + "log_vport"
+	devModeFlag           = optionPrefix + "devmode"
+	traceModeFlag         = optionPrefix + "trace"
+	useVsockFlag          = optionPrefix + "use_vsock"
+	debugConsoleFlag      = optionPrefix + "debug_console"
+	debugConsoleVPortFlag = optionPrefix + "debug_console_vport"
+	kernelCmdlineFile     = "/proc/cmdline"
+	traceModeStatic       = "static"
+	traceModeDynamic      = "dynamic"
+	traceTypeIsolated     = "isolated"
+	traceTypeCollated     = "collated"
+	defaultTraceType      = traceTypeIsolated
 )
 
 type agentConfig struct {
@@ -113,6 +114,13 @@ func (c *agentConfig) parseCmdlineOption(option string) error {
 			return err
 		}
 		logsVSockPort = uint32(port)
+	case debugConsoleVPortFlag:
+		port, err := strconv.ParseUint(split[valuePosition], 10, 32)
+		if err != nil {
+			return err
+		}
+		debugConsole = true
+		debugConsoleVSockPort = uint32(port)
 	case traceModeFlag:
 		switch split[valuePosition] {
 		case traceTypeIsolated:

config_test.go

@@ -366,3 +366,45 @@ func TestParseCmdlineOptionDebugConsole(t *testing.T) {
 		assert.True(debugConsole, "test %d (%+v)", i, d)
 	}
 }
+
+func TestParseCmdlineOptionDebugConsoleVPort(t *testing.T) {
+	assert := assert.New(t)
+
+	a := &agentConfig{}
+
+	type testData struct {
+		option                    string
+		expectDebugConsoleEnabled bool
+		expectedError             bool
+		expectedVPort             uint32
+	}
+
+	data := []testData{
+		{"", false, false, 0},
+		{"debug_console_vport", false, false, 0},
+		{"debug_console_vport=xxx", false, false, 0},
+		{"debug_console_vport=1026", false, false, 0},
+		{debugConsoleVPortFlag + "=", false, true, 0},
+		{debugConsoleVPortFlag + "=xxxx", false, true, 0},
+		{debugConsoleVPortFlag, false, false, 0},
+		{debugConsoleVPortFlag + "=1026", false, false, 1026},
+	}
+
+	for i, d := range data {
+		debugConsole = false
+		debugConsoleVSockPort = 0
+
+		err := a.parseCmdlineOption(d.option)
+		if d.expectedError {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+		}
+
+		if d.expectDebugConsoleEnabled {
+			assert.True(debugConsole, "test %d (%+v)", i, d)
+		}
+
+		assert.Equal(debugConsoleVSockPort, d.expectedVPort)
+	}
+}