Skip to content

Commit

Permalink
Implement TerminalCommand and Terminal command mode (#563)
Browse files Browse the repository at this point in the history
This change implements a new mode called `TERMINAL`. In this mode, the
server, right after starting the script, writes initial commands that
are responsible for collecting environment variables, which will be
added or deleted to the current session.

The `TERMINAL` mode is supported only in the `v2alpha1` runner.

Fixes #551
  • Loading branch information
adambabik authored May 6, 2024
1 parent e163a3a commit 0f8255d
Show file tree
Hide file tree
Showing 20 changed files with 590 additions and 204 deletions.
4 changes: 2 additions & 2 deletions experimental/runme.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ env:
# Available fields are defined in [config.FilterDocumentEnv] and [config.FilterBlockEnv].
filters:
# Do not allow unnamed code blocks.
- type: "FILTER_TYPE_BLOCK"
condition: "is_named"
# - type: "FILTER_TYPE_BLOCK"
# condition: "is_named"
# Do not allow code blocks without a language.
- type: "FILTER_TYPE_BLOCK"
condition: "language != ''"
Expand Down
1 change: 1 addition & 0 deletions internal/api/runme/runner/v1/runner.proto
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ enum CommandMode {
COMMAND_MODE_UNSPECIFIED = 0;
COMMAND_MODE_INLINE_SHELL = 1;
COMMAND_MODE_TEMP_FILE = 2;
COMMAND_MODE_TERMINAL = 3;
}

message Project {
Expand Down
1 change: 1 addition & 0 deletions internal/api/runme/runner/v2alpha1/config.proto
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ enum CommandMode {
COMMAND_MODE_UNSPECIFIED = 0;
COMMAND_MODE_INLINE = 1;
COMMAND_MODE_FILE = 2;
COMMAND_MODE_TERMINAL = 3;
}

// ProgramConfig is a configuration for a program to execute.
Expand Down
13 changes: 7 additions & 6 deletions internal/command/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ type Command interface {
}

type Options struct {
Kernel Kernel
Logger *zap.Logger
Session *Session
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
Kernel Kernel
Logger *zap.Logger
Session *Session
StdinWriter io.Writer
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
122 changes: 29 additions & 93 deletions internal/command/command_args_normalizer.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
package command

import (
"bufio"
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
Expand Down Expand Up @@ -32,12 +29,11 @@ var EnvDumpCommand = func() string {
}()

type argsNormalizer struct {
session *Session
logger *zap.Logger

tempDir string
isEnvCollectable bool
scriptFile *os.File
envCollector *shellEnvCollector
logger *zap.Logger
session *Session
scriptFile *os.File
tempDir string
}

func newArgsNormalizer(session *Session, logger *zap.Logger) configNormalizer {
Expand Down Expand Up @@ -94,6 +90,9 @@ func (n *argsNormalizer) Normalize(cfg *Config) (func() error, error) {
// TODO(adamb): it's not always true that the script-based program
// takes the filename as a last argument.
args = append(args, n.scriptFile.Name())

case *runnerv2alpha1.CommandMode_COMMAND_MODE_TERMINAL.Enum():
// noop
}

cfg.Arguments = args
Expand All @@ -107,18 +106,14 @@ func (n *argsNormalizer) inlineShell(cfg *Config, buf *strings.Builder) error {
_, _ = buf.WriteString("\n\n")
}

// If the session is provided, we need to collect the environment before and after the script execution.
// Here, we dump env before the script execution and use trap on EXIT to collect the env after the script execution.
// If the session is provided, the env should be collected.
if n.session != nil {
if err := n.createTempDir(); err != nil {
n.envCollector = &shellEnvCollector{
buf: buf,
}
if err := n.envCollector.Init(); err != nil {
return err
}

_, _ = buf.WriteString(fmt.Sprintf("%s > %s\n", EnvDumpCommand, filepath.Join(n.tempDir, envStartFileName)))
_, _ = buf.WriteString(fmt.Sprintf("__cleanup() {\nrv=$?\n%s > %s\nexit $rv\n}\n", EnvDumpCommand, filepath.Join(n.tempDir, envEndFileName)))
_, _ = buf.WriteString("trap -- \"__cleanup\" EXIT\n")

n.isEnvCollectable = true
}

// Write the script from the commands or the script.
Expand All @@ -134,13 +129,9 @@ func (n *argsNormalizer) inlineShell(cfg *Config, buf *strings.Builder) error {
return nil
}

func (n *argsNormalizer) cleanup() (result error) {
if err := n.collectEnv(); err != nil {
result = multierr.Append(result, err)
}
if err := n.removeTempDir(); err != nil {
result = multierr.Append(result, err)
}
func (n *argsNormalizer) createTempDir() (err error) {
n.tempDir, err = os.MkdirTemp("", "runme-*")
err = errors.WithMessage(err, "failed to create a temporary dir")
return
}

Expand All @@ -158,39 +149,27 @@ func (n *argsNormalizer) removeTempDir() error {
return nil
}

func (n *argsNormalizer) collectEnv() error {
if n.session == nil || !n.isEnvCollectable {
return nil
func (n *argsNormalizer) cleanup() (result error) {
if err := n.collectEnv(); err != nil {
result = multierr.Append(result, err)
}
if err := n.removeTempDir(); err != nil {
result = multierr.Append(result, err)
}
return
}

n.logger.Info("collecting env")

startEnv, err := n.readEnvFromFile(envStartFileName)
if err != nil {
return err
func (n *argsNormalizer) collectEnv() error {
if n.session == nil || n.envCollector == nil {
return nil
}

endEnv, err := n.readEnvFromFile(envEndFileName)
changed, deleted, err := n.envCollector.Collect()
if err != nil {
return err
}

// Below, we diff the env collected before and after the script execution.
// Then, update the session with the new or updated env and delete the deleted env.

startEnvStore := newEnvStore()
if _, err := startEnvStore.Merge(startEnv...); err != nil {
return errors.WithMessage(err, "failed to create the start env store")
}

endEnvStore := newEnvStore()
if _, err := endEnvStore.Merge(endEnv...); err != nil {
return errors.WithMessage(err, "failed to create the end env store")
}

newOrUpdated, _, deleted := diffEnvStores(startEnvStore, endEnvStore)

if err := n.session.SetEnv(newOrUpdated...); err != nil {
if err := n.session.SetEnv(changed...); err != nil {
return errors.WithMessage(err, "failed to set the new or updated env")
}

Expand All @@ -199,12 +178,6 @@ func (n *argsNormalizer) collectEnv() error {
return nil
}

func (n *argsNormalizer) createTempDir() (err error) {
n.tempDir, err = os.MkdirTemp("", "runme-*")
err = errors.WithMessage(err, "failed to create atemporery dir")
return
}

func (n *argsNormalizer) createScriptFile() (err error) {
n.scriptFile, err = os.CreateTemp(n.tempDir, "runme-script-*")
err = errors.WithMessage(err, "failed to create a temporary file for script execution")
Expand All @@ -218,43 +191,6 @@ func (n *argsNormalizer) writeScript(script []byte) error {
return errors.WithMessage(n.scriptFile.Close(), "failed to close the temporary file")
}

func (n *argsNormalizer) readEnvFromFile(name string) (result []string, _ error) {
f, err := os.Open(filepath.Join(n.tempDir, name))
if err != nil {
return nil, errors.WithMessagef(err, "failed to open the env file %q", name)
}
defer func() { _ = f.Close() }()

scanner := bufio.NewScanner(f)
scanner.Split(splitNull)

for scanner.Scan() {
result = append(result, scanner.Text())
}

if err := scanner.Err(); err != nil {
return nil, errors.WithMessagef(err, "failed to scan the env file %q", name)
}

return result, nil
}

func splitNull(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, 0); i >= 0 {
// We have a full null-terminated line.
return i + 1, data[0:i], nil
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
}

func shellOptionsFromProgram(programPath string) (res string) {
base := filepath.Base(programPath)
shell := base[:len(base)-len(filepath.Ext(base))]
Expand Down
67 changes: 67 additions & 0 deletions internal/command/command_terminal.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package command

import (
"context"

"github.com/pkg/errors"
)

type TerminalCommand struct {
*VirtualCommand

envCollector *shellEnvCollector
}

var _ Command = (*TerminalCommand)(nil)

func NewTerminal(cfg *Config, opts Options) *TerminalCommand {
return &TerminalCommand{
VirtualCommand: NewVirtual(cfg, opts),
}
}

func (c *TerminalCommand) Start(ctx context.Context) error {
if isNil(c.opts.StdinWriter) {
return errors.New("stdin writer is nil")
}

if err := c.VirtualCommand.Start(ctx); err != nil {
return err
}

c.opts.Logger.Info("a terminal command started")

c.envCollector = &shellEnvCollector{
buf: c.opts.StdinWriter,
}
return c.envCollector.Init()
}

func (c *TerminalCommand) Wait() (err error) {
err = c.VirtualCommand.Wait()

if cErr := c.collectEnv(); err == nil && cErr != nil {
err = cErr
}

return err
}

func (c *TerminalCommand) collectEnv() error {
if c.opts.Session == nil || c.envCollector == nil {
return nil
}

changed, deleted, err := c.envCollector.Collect()
if err != nil {
return err
}

if err := c.opts.Session.SetEnv(changed...); err != nil {
return errors.WithMessage(err, "failed to set the new or updated env")
}

c.opts.Session.DeleteEnv(deleted...)

return nil
}
71 changes: 71 additions & 0 deletions internal/command/command_terminal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
//go:build !windows

package command

import (
"bytes"
"context"
"io"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"

runnerv2alpha1 "github.com/stateful/runme/v3/internal/gen/proto/go/runme/runner/v2alpha1"
)

func TestTerminalCommand_Options_Stdinwriter_Nil(t *testing.T) {
cmd := NewTerminal(
&Config{
ProgramName: "bash",
Mode: runnerv2alpha1.CommandMode_COMMAND_MODE_TERMINAL,
},
Options{},
)

ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()

require.ErrorContains(t, cmd.Start(ctx), "stdin writer is nil")
}

func TestTerminalCommand(t *testing.T) {
logger := zaptest.NewLogger(t)
session := NewSession()

stdinR, stdinW := io.Pipe()
stdout := bytes.NewBuffer(nil)

cmd := NewTerminal(
&Config{
ProgramName: "bash",
Mode: runnerv2alpha1.CommandMode_COMMAND_MODE_TERMINAL,
},
Options{
Logger: logger,
Session: session,
StdinWriter: stdinW,
Stdin: stdinR,
Stdout: stdout,
},
)

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

require.NoError(t, cmd.Start(ctx))

// TODO(adamb): on macOS is is not necessary, but on Linux
// we need to wait for the shell to start before we start sending commands.
time.Sleep(time.Second)

_, err := stdinW.Write([]byte("export TEST_ENV=1\n"))
require.NoError(t, err)
_, err = stdinW.Write([]byte{0x04}) // EOT
require.NoError(t, err)

require.NoError(t, cmd.Wait())
assert.Equal(t, []string{"TEST_ENV=1"}, session.GetEnv())
}
8 changes: 8 additions & 0 deletions internal/command/command_virtual_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ func TestVirtualCommand(t *testing.T) {
assert.Equal(t, "test", stdout.String())
})

t.Run("BasicShell", func(t *testing.T) {
stdout := bytes.NewBuffer(nil)
cmd := NewVirtual(testConfigShellProgram, Options{Stdout: stdout})
require.NoError(t, cmd.Start(context.Background()))
require.NoError(t, cmd.Wait())
assert.Equal(t, "test", stdout.String())
})

t.Run("Getters", func(t *testing.T) {
cmd := NewVirtual(
&Config{
Expand Down
Loading

0 comments on commit 0f8255d

Please sign in to comment.