diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b448611d8..73041c05e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,13 @@ jobs: uses: denoland/setup-deno@v1 with: deno-version: v1.x + - name: Setup Dagger (Linux only) + uses: dagger/dagger-for-github@v7 + with: + version: "latest" + verb: core + args: "engine local-cache" + if: ${{ matrix.os == 'ubuntu-latest' }} - name: Setup go uses: actions/setup-go@v5 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58a642a0d..4e328bf5e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -128,7 +128,7 @@ python3 -m pip install pre-commit Like many complex go projects, this project uses a variety of linting tools to ensure code quality and prevent regressions! The main linter (revive) can be run with: -```sh {"id":"01HF7BT3HEQBTBM9SSTKQENPT3","interactive":"false"} +```sh {"id":"01HF7BT3HEQBTBM9SSTKQENPT3","interactive":"false","name":"lint"} make lint ``` diff --git a/examples/dagger/simple.dag b/examples/dagger/simple.dag new file mode 100644 index 000000000..adefa2830 --- /dev/null +++ b/examples/dagger/simple.dag @@ -0,0 +1,11 @@ +--- +shell: dagger shell +--- + +```sh {"name":"simple_dagger","terminalRows":"18"} +### Exported in runme.dev as simple_dagger +git github.com/stateful/runme | + head | + tree | + file examples/README.md +``` diff --git a/internal/command/config.go b/internal/command/config.go index a2efc8db0..53cd7ac9a 100644 --- a/internal/command/config.go +++ b/internal/command/config.go @@ -30,7 +30,7 @@ func isShell(cfg *ProgramConfig) bool { func isShellProgram(programName string) bool { switch strings.ToLower(programName) { - case "sh", "bash", "zsh", "ksh", "shell": + case "sh", "bash", "zsh", "ksh", "shell", "dagger shell": return true case "cmd", "powershell", "pwsh", "fish": return true diff --git a/internal/notebook/resolver.go b/internal/notebook/resolver.go index 3f84b7547..623397eba 100644 --- a/internal/notebook/resolver.go +++ b/internal/notebook/resolver.go @@ -3,7 +3,9 @@ package notebook import ( "bytes" "context" + "errors" "fmt" + "os" "strconv" "strings" @@ -11,6 +13,7 @@ import ( "github.com/stateful/runme/v3/internal/notebook/daggershell" parserv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1" + "github.com/stateful/runme/v3/pkg/document" "github.com/stateful/runme/v3/pkg/document/editor/editorservice" ) @@ -19,13 +22,54 @@ type NotebookResolver struct { editor parserv1.ParserServiceServer } -func NewNotebookResolver(notebook *parserv1.Notebook) *NotebookResolver { - return &NotebookResolver{ - notebook: notebook, - editor: editorservice.NewParserServiceServer(zap.NewNop()), +type Option func(*NotebookResolver) error + +func WithNotebook(notebook *parserv1.Notebook) Option { + return func(r *NotebookResolver) error { + r.notebook = notebook + return nil } } +func WithSource(source []byte) Option { + return func(r *NotebookResolver) error { + des, err := r.editor.Deserialize(context.Background(), &parserv1.DeserializeRequest{Source: source}) + if err != nil { + return err + } + + r.notebook = des.Notebook + return nil + } +} + +func WithDocumentPath(path string) Option { + return func(r *NotebookResolver) error { + source, err := os.ReadFile(path) + if err != nil { + return err + } + + return WithSource(source)(r) + } +} + +func NewResolver(opts ...Option) (*NotebookResolver, error) { + r := &NotebookResolver{ + editor: editorservice.NewParserServiceServer(zap.NewNop()), + } + + // apply options + for _, opt := range opts { + err := opt(r) + if err != nil { + return nil, err + } + } + + return r, nil +} + func (r *NotebookResolver) parseNotebook(context context.Context) (*parserv1.Notebook, error) { // make id sticky only for resolving purposes for _, cell := range r.notebook.Cells { @@ -125,3 +169,21 @@ func (r *NotebookResolver) ResolveDaggerShell(context context.Context, cellIndex return rendered.String(), nil } + +func (r *NotebookResolver) GetCellIndexByBlock(block *document.CodeBlock) (uint32, error) { + return getCellIndexByBlock(r.notebook, block) +} + +// todo(sebastian): there are better ways +func getCellIndexByBlock(notebook *parserv1.Notebook, block *document.CodeBlock) (blockIndex uint32, err error) { + blockIndex = 0 + for i, cell := range notebook.Cells { + blockValue := string(block.Content()) + if cell.Value == blockValue { + blockIndex = uint32(i) + return + } + } + + return blockIndex, errors.New("cell for block not found") +} diff --git a/internal/notebook/resolver_test.go b/internal/notebook/resolver_test.go index 752f77ddc..66437b5fa 100644 --- a/internal/notebook/resolver_test.go +++ b/internal/notebook/resolver_test.go @@ -8,8 +8,37 @@ import ( "github.com/stretchr/testify/require" parserv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1" + "github.com/stateful/runme/v3/pkg/document" + "github.com/stateful/runme/v3/pkg/document/identity" ) +func TestResolve_GetCellIndexByBlock(t *testing.T) { + simpleSource := []byte("---\nrunme:\n id: 01JJDCG2SQSGV0DP55XCR55AYM\n version: v3\nshell: dagger shell\nterminalRows: 20\n---\n\n# Compose Notebook Pipelines using the Dagger Shell\n\nLet's get upstream artifacts ready. First, compile the Runme kernel binary.\n\n```sh {\"id\":\"01JJDCG2SPRDWGQ1F4Z6EH69EJ\",\"name\":\"KERNEL_BINARY\"}\ngithub.com/purpleclay/daggerverse/golang $(git https://github.com/stateful/runme | head | tree) |\n build | \n file runme\n```\n\nThen, grab the presetup.sh script to provision the build container.\n\n```sh {\"id\":\"01JJDCG2SQSGV0DP55X86EJFSZ\",\"name\":\"PRESETUP\",\"terminalRows\":\"14\"}\ngit https://github.com/stateful/vscode-runme |\n head |\n tree |\n file dagger/scripts/presetup.sh\n```\n\n## Build the Runme VS Code Extension\n\nLet's tie together above's artifacts via their respective cell names to build the Runme VS Code extension.\n\n```sh {\"id\":\"01JJDCG2SQSGV0DP55X8JVYDNR\",\"name\":\"EXTENSION\",\"terminalRows\":\"25\"}\ngithub.com/stateful/vscode-runme |\n with-remote github.com/stateful/vscode-runme main |\n with-container $(KERNEL_BINARY) $(PRESETUP) |\n build-extension GITHUB_TOKEN\n```\n") + + resolver, err := NewResolver(WithSource(simpleSource)) + require.NoError(t, err) + + doc := document.New(simpleSource, identity.NewResolver(identity.DefaultLifecycleIdentity)) + require.NoError(t, err) + require.NotNil(t, doc) + + node, err := doc.Root() + require.NoError(t, err) + require.Len(t, node.Children(), 8) + + expectedValue := []byte("git https://github.com/stateful/vscode-runme |\n head |\n tree |\n file dagger/scripts/presetup.sh") + expectedIndex := uint32(4) + + block, ok := node.Children()[expectedIndex].Item().(*document.CodeBlock) + require.True(t, ok) + require.NotNil(t, block) + require.Equal(t, expectedValue, block.Content()) + + index, err := resolver.GetCellIndexByBlock(block) + require.NoError(t, err) + assert.Equal(t, expectedIndex, index) +} + func TestResolveDaggerShell(t *testing.T) { ctx := context.Background() @@ -55,7 +84,9 @@ func TestResolveDaggerShell(t *testing.T) { }, } - resolver := NewNotebookResolver(daggerShellNotebook) + resolver, err := NewResolver(WithNotebook(daggerShellNotebook)) + require.NoError(t, err) + definition := `KERNEL_BINARY() { github.com/purpleclay/daggerverse/golang $(git https://github.com/stateful/runme | head | tree) \ @@ -86,6 +117,18 @@ EXTENSION() } } +func TestResolveDaggerShell_Source(t *testing.T) { + simpleSource := "---\nshell: dagger shell\n---\n\n```sh {\"name\":\"simple_dagger\",\"terminalRows\":\"18\"}\n### Exported in runme.dev as simple_dagger\ngit github.com/stateful/runme |\n head |\n tree |\n file examples/README.md\n```\n" + + resolver, err := NewResolver(WithSource([]byte(simpleSource))) + require.NoError(t, err) + + script, err := resolver.ResolveDaggerShell(context.Background(), uint32(0)) + require.NoError(t, err) + + assert.Equal(t, "simple_dagger()\n{\n git github.com/stateful/runme \\\n | head \\\n | tree \\\n | file examples/README.md\n}\nsimple_dagger\n", script) +} + func TestResolveDaggerShell_EmptyRunmeMetadata(t *testing.T) { ctx := context.Background() @@ -104,7 +147,9 @@ func TestResolveDaggerShell_EmptyRunmeMetadata(t *testing.T) { }, } - resolver := NewNotebookResolver(daggerShellNotebook) + resolver, err := NewResolver(WithNotebook(daggerShellNotebook)) + require.NoError(t, err) + stub := `{ git github.com/stateful/runme \ | head \ diff --git a/internal/notebook/service.go b/internal/notebook/service.go index 4b900eca6..e3d60ea7c 100644 --- a/internal/notebook/service.go +++ b/internal/notebook/service.go @@ -44,7 +44,12 @@ func (r *notebookService) ResolveNotebook(ctx context.Context, req *notebookv1al return nil, status.Error(codes.InvalidArgument, "cell index is required") } - resolver := NewNotebookResolver(notebook) + resolver, err := NewResolver(WithNotebook(notebook)) + if err != nil { + r.logger.Error("failed to create notebook resolver", zap.Error(err)) + return nil, status.Error(codes.Internal, err.Error()) + } + script, err := resolver.ResolveDaggerShell(ctx, cellIndex.GetValue()) if err != nil { r.logger.Error("failed to resolve dagger shell", zap.Error(err)) diff --git a/internal/runner/client/client.go b/internal/runner/client/client.go index 1d9742121..c4a439382 100644 --- a/internal/runner/client/client.go +++ b/internal/runner/client/client.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" + "github.com/stateful/runme/v3/internal/notebook" "github.com/stateful/runme/v3/internal/runner" runnerv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1" "github.com/stateful/runme/v3/pkg/document" @@ -301,6 +302,35 @@ func ResolveDirectory(parentDir string, task project.Task) string { return parentDir } +func getCellProgram(languageID string, customShell string, task project.Task) (program string, lines []string, commandMode runner.CommandMode, err error) { + block := task.CodeBlock + lines = block.Lines() + + program, commandMode = runner.GetCellProgram(languageID, customShell, block) + if commandMode != runner.CommandModeDaggerShell { + return + } + + path := task.DocumentPath + resolver, err := notebook.NewResolver(notebook.WithDocumentPath(path)) + if err != nil { + return + } + + cellIndex, err := resolver.GetCellIndexByBlock(block) + if err != nil { + return + } + + script, err := resolver.ResolveDaggerShell(context.Background(), cellIndex) + if err != nil { + return + } + lines = strings.Split(script, "\n") + + return +} + func resolveOrAbsolute(parent string, child string) string { if child == "" { return parent diff --git a/internal/runner/client/client_local.go b/internal/runner/client/client_local.go index 6ad8fd1ea..9f2a85bcb 100644 --- a/internal/runner/client/client_local.go +++ b/internal/runner/client/client_local.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" + "github.com/stateful/runme/v3/internal/notebook" "github.com/stateful/runme/v3/internal/runner" runnerv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1" "github.com/stateful/runme/v3/pkg/document" @@ -22,15 +23,17 @@ import ( type LocalRunner struct { *RunnerSettings - shellID int - runnerService runnerv1.RunnerServiceServer + shellID int + runnerService runnerv1.RunnerServiceServer + notebookResolver notebook.NotebookResolver } func (r *LocalRunner) Clone() Runner { return &LocalRunner{ - RunnerSettings: r.RunnerSettings.Clone(), - shellID: r.shellID, - runnerService: r.runnerService, + RunnerSettings: r.RunnerSettings.Clone(), + shellID: r.shellID, + runnerService: r.runnerService, + notebookResolver: r.notebookResolver, } } @@ -100,7 +103,10 @@ func (r *LocalRunner) newExecutable(task project.Task) (runner.Executable, error customShell = fmtr.Shell } - programName, _ := runner.GetCellProgram(block.Language(), customShell, block) + programName, lines, _, err := getCellProgram(block.Language(), customShell, task) + if err != nil { + return nil, err + } cfg := &runner.ExecutableConfig{ Name: block.Name(), @@ -128,14 +134,14 @@ func (r *LocalRunner) newExecutable(task project.Task) (runner.Executable, error case "bash", "bat", "sh", "shell", "zsh", "": return &runner.Shell{ ExecutableConfig: cfg, - Cmds: block.Lines(), + Cmds: lines, CustomShell: customShell, }, nil case "sh-raw": return &runner.ShellRaw{ Shell: &runner.Shell{ ExecutableConfig: cfg, - Cmds: block.Lines(), + Cmds: lines, }, }, nil case "go": @@ -146,7 +152,7 @@ func (r *LocalRunner) newExecutable(task project.Task) (runner.Executable, error default: return &runner.TempFile{ ExecutableConfig: cfg, - Script: strings.Join(block.Lines(), "\n"), + Script: strings.Join(lines, "\n"), ProgramName: programName, LanguageID: block.Language(), }, nil diff --git a/internal/runner/client/client_remote.go b/internal/runner/client/client_remote.go index 7eb2c6196..7860d9063 100644 --- a/internal/runner/client/client_remote.go +++ b/internal/runner/client/client_remote.go @@ -204,7 +204,10 @@ func (r *RemoteRunner) RunTask(ctx context.Context, task project.Task) error { customShell = fmtr.Shell } - programName, commandMode := runner.GetCellProgram(block.Language(), customShell, block) + programName, lines, commandMode, err := getCellProgram(block.Language(), customShell, task) + if err != nil { + return err + } var commandModeGrpc runnerv1.CommandMode @@ -212,6 +215,7 @@ func (r *RemoteRunner) RunTask(ctx context.Context, task project.Task) error { case runner.CommandModeNone: commandModeGrpc = runnerv1.CommandMode_COMMAND_MODE_UNSPECIFIED case runner.CommandModeInlineShell: + case runner.CommandModeDaggerShell: commandModeGrpc = runnerv1.CommandMode_COMMAND_MODE_INLINE_SHELL case runner.CommandModeTempFile: commandModeGrpc = runnerv1.CommandMode_COMMAND_MODE_TEMP_FILE @@ -220,7 +224,7 @@ func (r *RemoteRunner) RunTask(ctx context.Context, task project.Task) error { req := &runnerv1.ExecuteRequest{ ProgramName: programName, Directory: r.dir, - Commands: block.Lines(), + Commands: lines, Tty: tty, SessionId: r.sessionID, SessionStrategy: r.sessionStrategy, diff --git a/internal/runner/shell.go b/internal/runner/shell.go index a979e316d..6c615b626 100644 --- a/internal/runner/shell.go +++ b/internal/runner/shell.go @@ -19,6 +19,8 @@ import ( "github.com/stateful/runme/v3/pkg/document" ) +const DaggerCustomShell = "dagger shell" + type Shell struct { *ExecutableConfig command *command @@ -136,10 +138,14 @@ func IsShellLanguage(languageID string) bool { } func GetCellProgram(languageID string, customShell string, cell *document.CodeBlock) (program string, commandMode CommandMode) { - if IsShellLanguage(languageID) { + switch { + case strings.Contains(customShell, DaggerCustomShell): + program = customShell + commandMode = CommandModeDaggerShell + case IsShellLanguage(languageID): program = customShell commandMode = CommandModeInlineShell - } else { + default: commandMode = CommandModeTempFile } @@ -152,8 +158,13 @@ func GetCellProgram(languageID string, customShell string, cell *document.CodeBl func resolveShellPath(customShell string) string { if customShell != "" { - if path, err := system.LookPath(customShell); err == nil { - return path + programParts := strings.Split(customShell, " ") + args := []string{} + if len(programParts) > 1 { + args = programParts[1:] + } + if path, err := system.LookPath(programParts[0]); err == nil { + return strings.Join(append([]string{path}, args...), " ") } } diff --git a/pkg/project/project.go b/pkg/project/project.go index 52de7bca2..72a4ce1db 100644 --- a/pkg/project/project.go +++ b/pkg/project/project.go @@ -518,7 +518,7 @@ func getCodeBlocks(data []byte) (document.CodeBlocks, error) { func isMarkdown(filePath string) bool { ext := strings.ToLower(filepath.Ext(filePath)) - return ext == ".md" || ext == ".mdx" || ext == ".mdi" || ext == ".mdr" || ext == ".run" || ext == ".runme" + return ext == ".md" || ext == ".mdx" || ext == ".mdi" || ext == ".mdr" || ext == ".run" || ext == ".runme" || ext == ".dag" } func (p *Project) LoadEnv() ([]string, error) { diff --git a/testdata/script/daggershell.txtar b/testdata/script/daggershell.txtar new file mode 100644 index 000000000..bda222f6f --- /dev/null +++ b/testdata/script/daggershell.txtar @@ -0,0 +1,42 @@ +exec runme ls +cmp stdout golden-list.txt +! stderr . + +exec runme ls --json +cmp stdout golden-list-json.txt +! stderr . + +exec runme run simple_dagger +stdout 'digest\: sha256' +stdout 'name\: README\.md' +stdout 'size\: ' + +-- shell.dag -- +--- +shell: dagger shell +--- + +```sh {"name":"simple_dagger","terminalRows":"18"} +### Exported in runme.dev as simple_dagger +git github.com/stateful/runme | + head | + tree | + file examples/README.md +``` + +-- golden-list.txt -- +NAME FILE FIRST COMMAND DESCRIPTION NAMED +simple_dagger* shell.dag git github.com/stateful/runme | Yes +-- golden-list-allow-unnamed.txt -- +NAME FILE FIRST COMMAND DESCRIPTION NAMED +-- golden-list-json.txt -- +[ + { + "name": "simple_dagger", + "file": "shell.dag", + "first_command": "git github.com/stateful/runme |", + "description": "", + "named": true, + "run_all": true + } +]