feat: support build time run without mount host (#1109)

* feat: support build time run without mount host Signed-off-by: Keming <kemingyang@tensorchord.ai> * add doc, fix lint Signed-off-by: Keming <kemingyang@tensorchord.ai> * Update envd/api/__init__.py Co-authored-by: Jinjing Zhou <VoVAllen@users.noreply.github.com> Signed-off-by: Keming <kemingy94@gmail.com> * address the review Signed-off-by: Keming <kemingyang@tensorchord.ai> Signed-off-by: Keming <kemingyang@tensorchord.ai> Signed-off-by: Keming <kemingy94@gmail.com> Co-authored-by: Jinjing Zhou <VoVAllen@users.noreply.github.com>
tensorchord · Oct 29, 2022 · f672c8f · f672c8f
1 parent 214f7c8
commit f672c8f
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 16 deletions.
diff --git a/envd/api/__init__.py b/envd/api/__init__.py
@@ -42,11 +42,13 @@ def shell(name: str):
     """
 
 
-def run(commands: str):
+def run(commands: str, mount_host: bool = False):
     """Execute command
 
     Args:
         commands (str): command to run during the building process
+        mount_host (bool): mount the host directory. Default is False.
+            Enabling this will disable the build cache for this operation.
 
     Example:
     ```

diff --git a/pkg/lang/frontend/starlark/universe/universe.go b/pkg/lang/frontend/starlark/universe/universe.go
@@ -62,9 +62,10 @@ func ruleFuncBase(thread *starlark.Thread, _ *starlark.Builtin,
 func ruleFuncRun(thread *starlark.Thread, _ *starlark.Builtin,
 	args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
 	var commands *starlark.List
+	mountHost := false
 
 	if err := starlark.UnpackArgs(ruleRun,
-		args, kwargs, "commands?", &commands); err != nil {
+		args, kwargs, "commands", &commands, "mount_host?", &mountHost); err != nil {
 		return nil, err
 	}
 
@@ -73,8 +74,8 @@ func ruleFuncRun(thread *starlark.Thread, _ *starlark.Builtin,
 		return nil, err
 	}
 
-	logger.Debugf("rule `%s` is invoked, commands=%v", ruleRun, goCommands)
-	if err := ir.Run(goCommands); err != nil {
+	logger.Debugf("rule `%s` is invoked, commands=%v, mount_host=%t", ruleRun, goCommands, mountHost)
+	if err := ir.Run(goCommands, mountHost); err != nil {
 		return nil, err
 	}
 

diff --git a/pkg/lang/ir/compile.go b/pkg/lang/ir/compile.go
@@ -56,7 +56,7 @@ func NewGraph() *Graph {
 		RPackages:       []string{},
 		JuliaPackages:   []string{},
 		SystemPackages:  []string{},
-		Exec:            [][]string{},
+		Exec:            []RunBuildCommand{},
 		UserDirectories: []string{},
 		Shell:           shellBASH,
 		CondaConfig:     conda,

diff --git a/pkg/lang/ir/interface.go b/pkg/lang/ir/interface.go
@@ -134,9 +134,11 @@ func RStudioServer() error {
 	return nil
 }
 
-func Run(commands []string) error {
-	// TODO(gaocegege): Support order-based exec.
-	DefaultGraph.Exec = append(DefaultGraph.Exec, commands)
+func Run(commands []string, mount bool) error {
+	DefaultGraph.Exec = append(DefaultGraph.Exec, RunBuildCommand{
+		Commands:  commands,
+		MountHost: mount,
+	})
 	return nil
 }
 

diff --git a/pkg/lang/ir/system.go b/pkg/lang/ir/system.go
@@ -52,11 +52,11 @@ func (g Graph) compileRun(root llb.State) llb.State {
 	}
 
 	workingDir := g.getWorkingDir()
-	stage := root
+	stage := root.AddEnv("PATH", types.DefaultPathEnvUnix)
 	for _, execGroup := range g.Exec {
 		var sb strings.Builder
 		sb.WriteString("set -euo pipefail\n")
-		for _, c := range execGroup {
+		for _, c := range execGroup.Commands {
 			sb.WriteString(c + "\n")
 		}
 
@@ -66,11 +66,11 @@ func (g Graph) compileRun(root llb.State) llb.State {
 		// TODO(gaocegege): Maybe we should make it readonly,
 		// but these cases then cannot be supported:
 		// run(commands=["git clone xx.git"])
-		stage = stage.
-			Run(llb.Shlex(cmdStr),
-				llb.AddEnv("PATH", types.DefaultPathEnvUnix),
-				llb.Dir(workingDir),
-				llb.AddMount(workingDir, llb.Local(flag.FlagBuildContext))).Root()
+		run := stage.Dir(workingDir).Run(llb.Shlex(cmdStr))
+		if execGroup.MountHost {
+			run.AddMount(workingDir, llb.Local(flag.FlagBuildContext))
+		}
+		stage = run.Root()
 	}
 	return stage
 }

diff --git a/pkg/lang/ir/types.go b/pkg/lang/ir/types.go
@@ -56,7 +56,7 @@ type Graph struct {
 	VSCodePlugins   []vscode.Plugin
 	UserDirectories []string
 
-	Exec       [][]string
+	Exec       []RunBuildCommand
 	Copy       []CopyInfo
 	Mount      []MountInfo
 	HTTP       []HTTPInfo
@@ -136,6 +136,11 @@ type JupyterConfig struct {
 	Port  int64
 }
 
+type RunBuildCommand struct {
+	Commands  []string
+	MountHost bool
+}
+
 const (
 	shellBASH = "bash"
 	shellZSH  = "zsh"