Dagger Shell natively in CLI (#745)

Natively run Dagger Shell when frontmatter (`shell: dagger shell`)
requires it. An example is available
[here](7c1b4b5).

Since the Notebook Resolver is entirely stateless we skip the GRPC
service layer. Please note that `txtar` e2e test coverage of the CLI is
excluded from SonarCloud.


https://github.com/user-attachments/assets/f7c273bd-a674-41cd-aa03-0e3cf648ef13

PS: The way Dagger's TUI renders in Runme's TUI isn't fantastic.
However, since no low-hanging fix was available this will have to wait
for a separate effort/PR.

.github/workflows/ci.yml

@@ -48,6 +48,13 @@ jobs:
         uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
+      - name: Setup Dagger (Linux only)
+        uses: dagger/dagger-for-github@v7
+        with:
+          version: "latest"
+          verb: core
+          args: "engine local-cache"
+        if: ${{ matrix.os == 'ubuntu-latest' }}
       - name: Setup go
         uses: actions/setup-go@v5
         with:

CONTRIBUTING.md

@@ -128,7 +128,7 @@ python3 -m pip install pre-commit
 
 Like many complex go projects, this project uses a variety of linting tools to ensure code quality and prevent regressions! The main linter (revive) can be run with:
 
-```sh {"id":"01HF7BT3HEQBTBM9SSTKQENPT3","interactive":"false"}
+```sh {"id":"01HF7BT3HEQBTBM9SSTKQENPT3","interactive":"false","name":"lint"}
 make lint
 ```
 

examples/dagger/simple.dag

@@ -0,0 +1,11 @@
+---
+shell: dagger shell
+---
+
+```sh {"name":"simple_dagger","terminalRows":"18"}
+### Exported in runme.dev as simple_dagger
+git github.com/stateful/runme |
+    head |
+    tree |
+    file examples/README.md
+```

internal/command/config.go

@@ -30,7 +30,7 @@ func isShell(cfg *ProgramConfig) bool {
 
 func isShellProgram(programName string) bool {
 	switch strings.ToLower(programName) {
-	case "sh", "bash", "zsh", "ksh", "shell":
+	case "sh", "bash", "zsh", "ksh", "shell", "dagger shell":
 		return true
 	case "cmd", "powershell", "pwsh", "fish":
 		return true

internal/notebook/resolver.go

@@ -3,14 +3,17 @@ package notebook
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
 	"go.uber.org/zap"
 
 	"github.com/stateful/runme/v3/internal/notebook/daggershell"
 	parserv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1"
+	"github.com/stateful/runme/v3/pkg/document"
 	"github.com/stateful/runme/v3/pkg/document/editor/editorservice"
 )
 
@@ -19,13 +22,54 @@ type NotebookResolver struct {
 	editor   parserv1.ParserServiceServer
 }
 
-func NewNotebookResolver(notebook *parserv1.Notebook) *NotebookResolver {
-	return &NotebookResolver{
-		notebook: notebook,
-		editor:   editorservice.NewParserServiceServer(zap.NewNop()),
+type Option func(*NotebookResolver) error
+
+func WithNotebook(notebook *parserv1.Notebook) Option {
+	return func(r *NotebookResolver) error {
+		r.notebook = notebook
+		return nil
 	}
 }
 
+func WithSource(source []byte) Option {
+	return func(r *NotebookResolver) error {
+		des, err := r.editor.Deserialize(context.Background(), &parserv1.DeserializeRequest{Source: source})
+		if err != nil {
+			return err
+		}
+
+		r.notebook = des.Notebook
+		return nil
+	}
+}
+
+func WithDocumentPath(path string) Option {
+	return func(r *NotebookResolver) error {
+		source, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		return WithSource(source)(r)
+	}
+}
+
+func NewResolver(opts ...Option) (*NotebookResolver, error) {
+	r := &NotebookResolver{
+		editor: editorservice.NewParserServiceServer(zap.NewNop()),
+	}
+
+	// apply options
+	for _, opt := range opts {
+		err := opt(r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return r, nil
+}
+
 func (r *NotebookResolver) parseNotebook(context context.Context) (*parserv1.Notebook, error) {
 	// make id sticky only for resolving purposes
 	for _, cell := range r.notebook.Cells {
@@ -125,3 +169,21 @@ func (r *NotebookResolver) ResolveDaggerShell(context context.Context, cellIndex
 
 	return rendered.String(), nil
 }
+
+func (r *NotebookResolver) GetCellIndexByBlock(block *document.CodeBlock) (uint32, error) {
+	return getCellIndexByBlock(r.notebook, block)
+}
+
+// todo(sebastian): there are better ways
+func getCellIndexByBlock(notebook *parserv1.Notebook, block *document.CodeBlock) (blockIndex uint32, err error) {
+	blockIndex = 0
+	for i, cell := range notebook.Cells {
+		blockValue := string(block.Content())
+		if cell.Value == blockValue {
+			blockIndex = uint32(i)
+			return
+		}
+	}
+
+	return blockIndex, errors.New("cell for block not found")
+}

internal/notebook/resolver_test.go

@@ -8,8 +8,37 @@ import (
 	"github.com/stretchr/testify/require"
 
 	parserv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1"
+	"github.com/stateful/runme/v3/pkg/document"
+	"github.com/stateful/runme/v3/pkg/document/identity"
 )
 
+func TestResolve_GetCellIndexByBlock(t *testing.T) {
+	simpleSource := []byte("---\nrunme:\n  id: 01JJDCG2SQSGV0DP55XCR55AYM\n  version: v3\nshell: dagger shell\nterminalRows: 20\n---\n\n# Compose Notebook Pipelines using the Dagger Shell\n\nLet's get upstream artifacts ready. First, compile the Runme kernel binary.\n\n```sh {\"id\":\"01JJDCG2SPRDWGQ1F4Z6EH69EJ\",\"name\":\"KERNEL_BINARY\"}\ngithub.com/purpleclay/daggerverse/golang $(git https://github.com/stateful/runme | head | tree) |\n  build | \n  file runme\n```\n\nThen, grab the presetup.sh script to provision the build container.\n\n```sh {\"id\":\"01JJDCG2SQSGV0DP55X86EJFSZ\",\"name\":\"PRESETUP\",\"terminalRows\":\"14\"}\ngit https://github.com/stateful/vscode-runme |\n  head |\n  tree |\n  file dagger/scripts/presetup.sh\n```\n\n## Build the Runme VS Code Extension\n\nLet's tie together above's artifacts via their respective cell names to build the Runme VS Code extension.\n\n```sh {\"id\":\"01JJDCG2SQSGV0DP55X8JVYDNR\",\"name\":\"EXTENSION\",\"terminalRows\":\"25\"}\ngithub.com/stateful/vscode-runme |\n  with-remote github.com/stateful/vscode-runme main |\n  with-container $(KERNEL_BINARY) $(PRESETUP) |\n  build-extension GITHUB_TOKEN\n```\n")
+
+	resolver, err := NewResolver(WithSource(simpleSource))
+	require.NoError(t, err)
+
+	doc := document.New(simpleSource, identity.NewResolver(identity.DefaultLifecycleIdentity))
+	require.NoError(t, err)
+	require.NotNil(t, doc)
+
+	node, err := doc.Root()
+	require.NoError(t, err)
+	require.Len(t, node.Children(), 8)
+
+	expectedValue := []byte("git https://github.com/stateful/vscode-runme |\n  head |\n  tree |\n  file dagger/scripts/presetup.sh")
+	expectedIndex := uint32(4)
+
+	block, ok := node.Children()[expectedIndex].Item().(*document.CodeBlock)
+	require.True(t, ok)
+	require.NotNil(t, block)
+	require.Equal(t, expectedValue, block.Content())
+
+	index, err := resolver.GetCellIndexByBlock(block)
+	require.NoError(t, err)
+	assert.Equal(t, expectedIndex, index)
+}
+
 func TestResolveDaggerShell(t *testing.T) {
 	ctx := context.Background()
 
@@ -55,7 +84,9 @@ func TestResolveDaggerShell(t *testing.T) {
 		},
 	}
 
-	resolver := NewNotebookResolver(daggerShellNotebook)
+	resolver, err := NewResolver(WithNotebook(daggerShellNotebook))
+	require.NoError(t, err)
+
 	definition := `KERNEL_BINARY()
 {
   github.com/purpleclay/daggerverse/golang $(git https://github.com/stateful/runme | head | tree) \
@@ -86,6 +117,18 @@ EXTENSION()
 	}
 }
 
+func TestResolveDaggerShell_Source(t *testing.T) {
+	simpleSource := "---\nshell: dagger shell\n---\n\n```sh {\"name\":\"simple_dagger\",\"terminalRows\":\"18\"}\n### Exported in runme.dev as simple_dagger\ngit github.com/stateful/runme |\n    head |\n    tree |\n    file examples/README.md\n```\n"
+
+	resolver, err := NewResolver(WithSource([]byte(simpleSource)))
+	require.NoError(t, err)
+
+	script, err := resolver.ResolveDaggerShell(context.Background(), uint32(0))
+	require.NoError(t, err)
+
+	assert.Equal(t, "simple_dagger()\n{\n  git github.com/stateful/runme \\\n    | head \\\n    | tree \\\n    | file examples/README.md\n}\nsimple_dagger\n", script)
+}
+
 func TestResolveDaggerShell_EmptyRunmeMetadata(t *testing.T) {
 	ctx := context.Background()
 
@@ -104,7 +147,9 @@ func TestResolveDaggerShell_EmptyRunmeMetadata(t *testing.T) {
 		},
 	}
 
-	resolver := NewNotebookResolver(daggerShellNotebook)
+	resolver, err := NewResolver(WithNotebook(daggerShellNotebook))
+	require.NoError(t, err)
+
 	stub := `{
   git github.com/stateful/runme \
     | head \

internal/notebook/service.go

@@ -44,7 +44,12 @@ func (r *notebookService) ResolveNotebook(ctx context.Context, req *notebookv1al
 		return nil, status.Error(codes.InvalidArgument, "cell index is required")
 	}
 
-	resolver := NewNotebookResolver(notebook)
+	resolver, err := NewResolver(WithNotebook(notebook))
+	if err != nil {
+		r.logger.Error("failed to create notebook resolver", zap.Error(err))
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
 	script, err := resolver.ResolveDaggerShell(ctx, cellIndex.GetValue())
 	if err != nil {
 		r.logger.Error("failed to resolve dagger shell", zap.Error(err))

internal/runner/client/client.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
+	"github.com/stateful/runme/v3/internal/notebook"
 	"github.com/stateful/runme/v3/internal/runner"
 	runnerv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1"
 	"github.com/stateful/runme/v3/pkg/document"
@@ -301,6 +302,35 @@ func ResolveDirectory(parentDir string, task project.Task) string {
 	return parentDir
 }
 
+func getCellProgram(languageID string, customShell string, task project.Task) (program string, lines []string, commandMode runner.CommandMode, err error) {
+	block := task.CodeBlock
+	lines = block.Lines()
+
+	program, commandMode = runner.GetCellProgram(languageID, customShell, block)
+	if commandMode != runner.CommandModeDaggerShell {
+		return
+	}
+
+	path := task.DocumentPath
+	resolver, err := notebook.NewResolver(notebook.WithDocumentPath(path))
+	if err != nil {
+		return
+	}
+
+	cellIndex, err := resolver.GetCellIndexByBlock(block)
+	if err != nil {
+		return
+	}
+
+	script, err := resolver.ResolveDaggerShell(context.Background(), cellIndex)
+	if err != nil {
+		return
+	}
+	lines = strings.Split(script, "\n")
+
+	return
+}
+
 func resolveOrAbsolute(parent string, child string) string {
 	if child == "" {
 		return parent

internal/runner/client/client_local.go

@@ -13,6 +13,7 @@ import (
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
+	"github.com/stateful/runme/v3/internal/notebook"
 	"github.com/stateful/runme/v3/internal/runner"
 	runnerv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1"
 	"github.com/stateful/runme/v3/pkg/document"
@@ -22,15 +23,17 @@ import (
 type LocalRunner struct {
 	*RunnerSettings
 
-	shellID       int
-	runnerService runnerv1.RunnerServiceServer
+	shellID          int
+	runnerService    runnerv1.RunnerServiceServer
+	notebookResolver notebook.NotebookResolver
 }
 
 func (r *LocalRunner) Clone() Runner {
 	return &LocalRunner{
-		RunnerSettings: r.RunnerSettings.Clone(),
-		shellID:        r.shellID,
-		runnerService:  r.runnerService,
+		RunnerSettings:   r.RunnerSettings.Clone(),
+		shellID:          r.shellID,
+		runnerService:    r.runnerService,
+		notebookResolver: r.notebookResolver,
 	}
 }
 
@@ -100,7 +103,10 @@ func (r *LocalRunner) newExecutable(task project.Task) (runner.Executable, error
 		customShell = fmtr.Shell
 	}
 
-	programName, _ := runner.GetCellProgram(block.Language(), customShell, block)
+	programName, lines, _, err := getCellProgram(block.Language(), customShell, task)
+	if err != nil {
+		return nil, err
+	}
 
 	cfg := &runner.ExecutableConfig{
 		Name:    block.Name(),
@@ -128,14 +134,14 @@ func (r *LocalRunner) newExecutable(task project.Task) (runner.Executable, error
 	case "bash", "bat", "sh", "shell", "zsh", "":
 		return &runner.Shell{
 			ExecutableConfig: cfg,
-			Cmds:             block.Lines(),
+			Cmds:             lines,
 			CustomShell:      customShell,
 		}, nil
 	case "sh-raw":
 		return &runner.ShellRaw{
 			Shell: &runner.Shell{
 				ExecutableConfig: cfg,
-				Cmds:             block.Lines(),
+				Cmds:             lines,
 			},
 		}, nil
 	case "go":
@@ -146,7 +152,7 @@ func (r *LocalRunner) newExecutable(task project.Task) (runner.Executable, error
 	default:
 		return &runner.TempFile{
 			ExecutableConfig: cfg,
-			Script:           strings.Join(block.Lines(), "\n"),
+			Script:           strings.Join(lines, "\n"),
 			ProgramName:      programName,
 			LanguageID:       block.Language(),
 		}, nil

internal/runner/client/client_remote.go

@@ -204,14 +204,18 @@ func (r *RemoteRunner) RunTask(ctx context.Context, task project.Task) error {
 		customShell = fmtr.Shell
 	}
 
-	programName, commandMode := runner.GetCellProgram(block.Language(), customShell, block)
+	programName, lines, commandMode, err := getCellProgram(block.Language(), customShell, task)
+	if err != nil {
+		return err
+	}
 
 	var commandModeGrpc runnerv1.CommandMode
 
 	switch commandMode {
 	case runner.CommandModeNone:
 		commandModeGrpc = runnerv1.CommandMode_COMMAND_MODE_UNSPECIFIED
 	case runner.CommandModeInlineShell:
+	case runner.CommandModeDaggerShell:
 		commandModeGrpc = runnerv1.CommandMode_COMMAND_MODE_INLINE_SHELL
 	case runner.CommandModeTempFile:
 		commandModeGrpc = runnerv1.CommandMode_COMMAND_MODE_TEMP_FILE
@@ -220,7 +224,7 @@ func (r *RemoteRunner) RunTask(ctx context.Context, task project.Task) error {
 	req := &runnerv1.ExecuteRequest{
 		ProgramName:     programName,
 		Directory:       r.dir,
-		Commands:        block.Lines(),
+		Commands:        lines,
 		Tty:             tty,
 		SessionId:       r.sessionID,
 		SessionStrategy: r.sessionStrategy,