From 06df26d47c6fb42935a7c2983b1b8b0b5af6dcff Mon Sep 17 00:00:00 2001 From: Keming Date: Mon, 15 Aug 2022 22:22:20 +0800 Subject: [PATCH 1/2] add daemon Signed-off-by: Keming --- examples/python-basic/build.envd | 6 ++++++ pkg/builder/util.go | 2 +- pkg/lang/frontend/starlark/runtime/const.go | 1 + pkg/lang/frontend/starlark/runtime/runtime.go | 21 +++++++++++++++++++ pkg/lang/ir/compile.go | 8 +++++-- pkg/lang/ir/interface.go | 4 ++++ pkg/lang/ir/types.go | 1 + 7 files changed, 40 insertions(+), 3 deletions(-) diff --git a/examples/python-basic/build.envd b/examples/python-basic/build.envd index e8d345710..7b3d32835 100644 --- a/examples/python-basic/build.envd +++ b/examples/python-basic/build.envd @@ -1,3 +1,8 @@ +def jupyter_lab(): + install.python_packages(["jupyterlab"]) + runtime.daemon(commands=["jupyter-lab"]) + + def build(): base(os="ubuntu20.04", language="python") #config.pip_index(url = "https://pypi.tuna.tsinghua.edu.cn/simple") @@ -8,3 +13,4 @@ def build(): runtime.command(commands={ "test": "ls /", }) + jupyter_lab() diff --git a/pkg/builder/util.go b/pkg/builder/util.go index c92093f0d..0bb246cd8 100644 --- a/pkg/builder/util.go +++ b/pkg/builder/util.go @@ -63,7 +63,7 @@ func ImageConfigStr(labels map[string]string, // DefaultPathEnvUnix is unix style list of directories to search for // executables. Each directory is separated from the next by a colon // ':' character . -const DefaultPathEnvUnix = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/conda/bin:/usr/local/julia/bin" +const DefaultPathEnvUnix = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/conda/bin:/opt/conda/envs/envd/bin/:/usr/local/julia/bin" // DefaultPathEnvWindows is windows style list of directories to search for // executables. Each directory is separated from the next by a colon diff --git a/pkg/lang/frontend/starlark/runtime/const.go b/pkg/lang/frontend/starlark/runtime/const.go index 8feeba927..9e7c70fe2 100644 --- a/pkg/lang/frontend/starlark/runtime/const.go +++ b/pkg/lang/frontend/starlark/runtime/const.go @@ -16,4 +16,5 @@ package runtime const ( ruleCommand = "runtime.command" + ruleDaemon = "runtime.daemon" ) diff --git a/pkg/lang/frontend/starlark/runtime/runtime.go b/pkg/lang/frontend/starlark/runtime/runtime.go index 31abb554f..dcbfd26d4 100644 --- a/pkg/lang/frontend/starlark/runtime/runtime.go +++ b/pkg/lang/frontend/starlark/runtime/runtime.go @@ -32,6 +32,7 @@ var Module = &starlarkstruct.Module{ Name: "runtime", Members: starlark.StringDict{ "command": starlark.NewBuiltin(ruleCommand, ruleFuncCommand), + "daemon": starlark.NewBuiltin(ruleDaemon, ruleFuncDaemon), }, } @@ -60,3 +61,23 @@ func ruleFuncCommand(thread *starlark.Thread, _ *starlark.Builtin, ir.RuntimeCommands(commandsMap) return starlark.None, nil } + +func ruleFuncDaemon(thread *starlark.Thread, _ *starlark.Builtin, + args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var commands *starlark.List + + if err := starlark.UnpackArgs(ruleDaemon, args, kwargs, "commands", &commands); err != nil { + return nil, err + } + + commandList := []string{} + if commands != nil { + for i := 0; i < commands.Len(); i++ { + commandList = append(commandList, commands.Index(i).(starlark.String).GoString()) + } + + logger.Debugf("rule `%s` is invoked, commands=%v", ruleDaemon, commandList) + ir.RuntimeDaemon(commandList) + } + return starlark.None, nil +} diff --git a/pkg/lang/ir/compile.go b/pkg/lang/ir/compile.go index bb728b5bf..2f003d911 100644 --- a/pkg/lang/ir/compile.go +++ b/pkg/lang/ir/compile.go @@ -177,14 +177,18 @@ wait -n` // Generate jupyter and rstudio server commands. var customCmd strings.Builder + workingDir := filepath.Join("/home/envd", filepath.Base(buildContextDir)) + if g.RuntimeDaemon != nil { + for _, command := range g.RuntimeDaemon { + customCmd.WriteString(fmt.Sprintf("%s -c %s &\n", g.Shell, command)) + } + } if g.JupyterConfig != nil { - workingDir := filepath.Join("/home/envd", filepath.Base(buildContextDir)) jupyterCmd := g.generateJupyterCommand(workingDir) customCmd.WriteString(strings.Join(jupyterCmd, " ")) customCmd.WriteString("\n") } if g.RStudioServerConfig != nil { - workingDir := filepath.Join("/home/envd", filepath.Base(buildContextDir)) rstudioCmd := g.generateRStudioCommand(workingDir) customCmd.WriteString(strings.Join(rstudioCmd, " ")) customCmd.WriteString("\n") diff --git a/pkg/lang/ir/interface.go b/pkg/lang/ir/interface.go index 9c7a764ac..ba194eff4 100644 --- a/pkg/lang/ir/interface.go +++ b/pkg/lang/ir/interface.go @@ -213,3 +213,7 @@ func RuntimeCommands(commands map[string]string) { DefaultGraph.RuntimeCommands[k] = v } } + +func RuntimeDaemon(commands []string) { + DefaultGraph.RuntimeDaemon = append(DefaultGraph.RuntimeDaemon, commands...) +} diff --git a/pkg/lang/ir/types.go b/pkg/lang/ir/types.go index ae1c08162..c35396272 100644 --- a/pkg/lang/ir/types.go +++ b/pkg/lang/ir/types.go @@ -55,6 +55,7 @@ type Graph struct { Mount []MountInfo Entrypoint []string RuntimeCommands map[string]string + RuntimeDaemon []string *JupyterConfig *GitConfig From a450d4a0d402565e88a55173be7c03e5c925c101 Mon Sep 17 00:00:00 2001 From: Keming Date: Tue, 16 Aug 2022 17:20:54 +0800 Subject: [PATCH 2/2] change to list of list string Signed-off-by: Keming --- examples/python-basic/build.envd | 4 +++- pkg/lang/frontend/starlark/runtime/runtime.go | 13 +++++++++++-- pkg/lang/ir/compile.go | 2 +- pkg/lang/ir/interface.go | 2 +- pkg/lang/ir/types.go | 2 +- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/examples/python-basic/build.envd b/examples/python-basic/build.envd index 7b3d32835..ec29788e9 100644 --- a/examples/python-basic/build.envd +++ b/examples/python-basic/build.envd @@ -1,6 +1,8 @@ def jupyter_lab(): install.python_packages(["jupyterlab"]) - runtime.daemon(commands=["jupyter-lab"]) + runtime.daemon(commands=[ + ["jupyter-lab", "--port", "8080"], + ]) def build(): diff --git a/pkg/lang/frontend/starlark/runtime/runtime.go b/pkg/lang/frontend/starlark/runtime/runtime.go index dcbfd26d4..8bcac39d2 100644 --- a/pkg/lang/frontend/starlark/runtime/runtime.go +++ b/pkg/lang/frontend/starlark/runtime/runtime.go @@ -70,10 +70,19 @@ func ruleFuncDaemon(thread *starlark.Thread, _ *starlark.Builtin, return nil, err } - commandList := []string{} + commandList := [][]string{} if commands != nil { for i := 0; i < commands.Len(); i++ { - commandList = append(commandList, commands.Index(i).(starlark.String).GoString()) + args, ok := commands.Index(i).(*starlark.List) + if !ok { + logger.Warnf("cannot parse %s into a list of string", commands.Index(i).String()) + continue + } + argList := []string{} + for j := 0; j < args.Len(); j++ { + argList = append(argList, args.Index(j).(starlark.String).GoString()) + } + commandList = append(commandList, argList) } logger.Debugf("rule `%s` is invoked, commands=%v", ruleDaemon, commandList) diff --git a/pkg/lang/ir/compile.go b/pkg/lang/ir/compile.go index 2f003d911..8dab31a56 100644 --- a/pkg/lang/ir/compile.go +++ b/pkg/lang/ir/compile.go @@ -180,7 +180,7 @@ wait -n` workingDir := filepath.Join("/home/envd", filepath.Base(buildContextDir)) if g.RuntimeDaemon != nil { for _, command := range g.RuntimeDaemon { - customCmd.WriteString(fmt.Sprintf("%s -c %s &\n", g.Shell, command)) + customCmd.WriteString(fmt.Sprintf("%s -c %s &\n", g.Shell, strings.Join(command, " "))) } } if g.JupyterConfig != nil { diff --git a/pkg/lang/ir/interface.go b/pkg/lang/ir/interface.go index ba194eff4..7f23e3e0e 100644 --- a/pkg/lang/ir/interface.go +++ b/pkg/lang/ir/interface.go @@ -214,6 +214,6 @@ func RuntimeCommands(commands map[string]string) { } } -func RuntimeDaemon(commands []string) { +func RuntimeDaemon(commands [][]string) { DefaultGraph.RuntimeDaemon = append(DefaultGraph.RuntimeDaemon, commands...) } diff --git a/pkg/lang/ir/types.go b/pkg/lang/ir/types.go index c35396272..c510896e6 100644 --- a/pkg/lang/ir/types.go +++ b/pkg/lang/ir/types.go @@ -55,7 +55,7 @@ type Graph struct { Mount []MountInfo Entrypoint []string RuntimeCommands map[string]string - RuntimeDaemon []string + RuntimeDaemon [][]string *JupyterConfig *GitConfig