feat(build): Mount local build context into the run command (#822)

* feat(build): Mount local build context into the run command

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* feat: Support build context in pip

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Update

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Update

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* Update pkg/lang/ir/system.go

Co-authored-by: Keming <kemingy94@gmail.com>

* Update pkg/lang/ir/system.go

Co-authored-by: Keming <kemingy94@gmail.com>

Signed-off-by: Ce Gao <cegao@tensorchord.ai>
Co-authored-by: Keming <kemingy94@gmail.com>

e2e/python_test.go

@@ -0,0 +1,49 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+ . "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("python", Ordered, func() {
+ It("Should build packages successfully", func() {
+ exampleName := "python/packages"
+ testcase := "e2e"
+ e := NewExample(exampleName, testcase)
+ e.BuildImage(true)()
+ e.RunContainer()()
+ e.DestroyContainer()()
+ e.RemoveImage()()
+ })
+ It("Should build requirements successfully", func() {
+ exampleName := "python/requirements"
+ testcase := "e2e"
+ e := NewExample(exampleName, testcase)
+ e.BuildImage(true)()
+ e.RunContainer()()
+ e.DestroyContainer()()
+ e.RemoveImage()()
+ })
+ It("Should build hybrid successfully", func() {
+ exampleName := "python/hybrid"
+ testcase := "e2e"
+ e := NewExample(exampleName, testcase)
+ e.BuildImage(true)()
+ e.RunContainer()()
+ e.DestroyContainer()()
+ e.RemoveImage()()
+ })
+})

e2e/testdata/python/hybrid/build.envd

@@ -0,0 +1,4 @@
+def build():
+ install.python_packages(name=[
+ "via"
+ ], requirements="requirements.txt")

e2e/testdata/python/hybrid/requirements.txt

@@ -0,0 +1 @@
+via

e2e/testdata/python/packages/build.envd

@@ -0,0 +1,4 @@
+def build():
+ install.python_packages(name = [
+ "via"
+ ])

e2e/testdata/python/requirements/build.envd

@@ -0,0 +1,2 @@
+def build():
+ install.python_packages(requirements="./requirements.txt")

e2e/testdata/python/requirements/requirements.txt

@@ -0,0 +1 @@
+via

e2e/testdata/run/build.envd

@@ -0,0 +1,9 @@
+def build():
+ base(os="ubuntu20.04", language="python3")
+ shell("zsh")
+ run(commands=[
+ "mkdir test",
+ "cd test",
+ "ls",
+ "pwd",
+ ])

examples/ianvs/build.envd

@@ -0,0 +1,11 @@
+def build():
+ base(os="ubuntu20.04", language="python3.6")
+ shell("zsh")
+ install.system_packages(name=["git", "libgl1-mesa-glx", "zip"])
+ run(commands=[
+ "git clone https://github.com/kubeedge/ianvs.git",
+ "cd ./ianvs",
+ "pip install -r requirements.txt",
+ "pip install ./examples/resources/third_party/*",
+ "python setup.py install"
+ ])

pkg/builder/builder.go

@@ -176,7 +176,8 @@ func (b generalBuilder) Interpret() error {
 }
 
 func (b generalBuilder) compile(ctx context.Context) (*llb.Definition, error) {
- def, err := ir.Compile(ctx, filepath.Base(b.BuildContextDir), b.PubKeyPath)
+ envName := filepath.Base(b.BuildContextDir)
+ def, err := ir.Compile(ctx, envName, b.PubKeyPath)
  if err != nil {
  return nil, errors.Wrap(err, "failed to compile build.envd")
  }

pkg/lang/frontend/starlark/install/install.go

@@ -59,19 +59,12 @@ func ruleFuncPyPIPackage(thread *starlark.Thread, _ *starlark.Builtin,
  }
  }
 
- var path *string = nil
  requirementsFileStr := requirementsFile.GoString()
- if requirementsFileStr != "" {
- buildContextDir := starlark.Universe[builtin.BuildContextDir]
- buildContextDirStr := buildContextDir.(starlark.String).GoString()
- buf := filepath.Join(buildContextDirStr, requirementsFileStr)
- path = &buf
- }
 
  logger.Debugf("rule `%s` is invoked, name=%v, requirements=%s",
  rulePyPIPackage, nameList, requirementsFileStr)
 
- err := ir.PyPIPackage(nameList, path)
+ err := ir.PyPIPackage(nameList, requirementsFileStr)
  return starlark.None, err
 }
 

pkg/lang/ir/cache.go

@@ -24,9 +24,9 @@ func (g Graph) CacheID(filename string) string {
  gpu := g.CUDA != nil || g.CUDNN != nil
  var cacheID string
  if gpu {
- cacheID = fmt.Sprintf("%s/%s-gpu", filename, g.CachePrefix)
+ cacheID = fmt.Sprintf("%s/%s-gpu", filename, g.EnvironmentName)
  } else {
- cacheID = fmt.Sprintf("%s/%s-cpu", filename, g.CachePrefix)
+ cacheID = fmt.Sprintf("%s/%s-cpu", filename, g.EnvironmentName)
  }
  logrus.Debugf("apt/pypi calculated cacheID: %s", cacheID)
  return cacheID

pkg/lang/ir/compile.go

@@ -39,10 +39,12 @@ func NewGraph() *Graph {
  RuntimeCommands: make(map[string]string),
  RuntimeEnviron: make(map[string]string),
  }
+ langVersion := languageVersionDefault
  return &Graph{
  OS: osDefault,
  Language: Language{
- Name: languageDefault,
+ Name: languageDefault,
+ Version: &langVersion,
  },
  CUDA: nil,
  CUDNN: nil,
@@ -68,13 +70,13 @@ func NumGPUs() int {
  return DefaultGraph.NumGPUs
 }
 
-func Compile(ctx context.Context, cachePrefix string, pub string) (*llb.Definition, error) {
+func Compile(ctx context.Context, envName string, pub string) (*llb.Definition, error) {
  w, err := compileui.New(ctx, os.Stdout, "auto")
  if err != nil {
  return nil, errors.Wrap(err, "failed to create compileui")
  }
  DefaultGraph.Writer = w
- DefaultGraph.CachePrefix = cachePrefix
+ DefaultGraph.EnvironmentName = envName
  DefaultGraph.PublicKeyPath = pub
 
  uid, gid, err := getUIDGID()
@@ -86,7 +88,11 @@ func Compile(ctx context.Context, cachePrefix string, pub string) (*llb.Definiti
  return nil, errors.Wrap(err, "failed to compile")
  }
  // TODO(gaocegege): Support multi platform.
- return state.Marshal(ctx, llb.LinuxAmd64)
+ def, err := state.Marshal(ctx, llb.LinuxAmd64)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to marshal the llb definition")
+ }
+ return def, nil
 }
 
 func Labels() (map[string]string, error) {

pkg/lang/ir/consts.go

@@ -15,9 +15,10 @@
 package ir
 
 const (
- osDefault = "ubuntu20.04"
- languageDefault = "python3"
- pypiIndexModeAuto = "auto"
+ osDefault = "ubuntu20.04"
+ languageDefault = "python"
+ languageVersionDefault = "3"
+ pypiIndexModeAuto = "auto"
 
  // used inside the container
  defaultConfigDir = "/home/envd/.config"

pkg/lang/ir/interface.go

@@ -37,16 +37,13 @@ func Base(os, language, image string) error {
  return nil
 }
 
-func PyPIPackage(deps []string, requirementsFile *string) error {
+func PyPIPackage(deps []string, requirementsFile string) error {
  DefaultGraph.PyPIPackages = append(DefaultGraph.PyPIPackages, deps...)
 
- if requirementsFile != nil {
- parsed, err := parser.ParsePythonRequirements(*requirementsFile)
- if err != nil {
- return err
- }
- DefaultGraph.PyPIPackages = append(DefaultGraph.PyPIPackages, parsed...)
+ if requirementsFile != "" {
+ DefaultGraph.RequirementsFile = &requirementsFile
  }
+
  return nil
 }
 

pkg/lang/ir/python.go

@@ -22,6 +22,8 @@ import (
  "github.com/cockroachdb/errors"
  "github.com/moby/buildkit/client/llb"
  "github.com/sirupsen/logrus"
+
+ "github.com/tensorchord/envd/pkg/flag"
 )
 
 const (
@@ -113,34 +115,58 @@ func (g Graph) compileAlternative(root llb.State) llb.State {
 }
 
 func (g Graph) compilePyPIPackages(root llb.State) llb.State {
- if len(g.PyPIPackages) == 0 {
+ if len(g.PyPIPackages) == 0 && g.RequirementsFile == nil {
  return root
  }
 
  cacheDir := "/home/envd/.cache"
  // Create the cache directory to the container. see issue #582
  root = g.CompileCacheDir(root, cacheDir)
 
- // Compose the package install command.
- var sb strings.Builder
- // Always use the conda's pip.
- sb.WriteString("/opt/conda/envs/envd/bin/python -m pip install")
- for _, pkg := range g.PyPIPackages {
- sb.WriteString(fmt.Sprintf(" %s", pkg))
- }
-
- cmd := sb.String()
- logrus.Debugf("pip command: %s", cmd)
- root = llb.User("envd")(root)
- // Refer to https://github.com/moby/buildkit/blob/31054718bf775bf32d1376fe1f3611985f837584/frontend/dockerfile/dockerfile2llb/convert_runmount.go#L46
  cache := root.File(llb.Mkdir("/cache",
  0755, llb.WithParents(true), llb.WithUIDGID(g.uid, g.gid)), llb.WithCustomName("[internal] setting pip cache mount permissions"))
- run := root.
- Run(llb.Shlex(cmd), llb.WithCustomNamef("pip install %s",
- strings.Join(g.PyPIPackages, " ")))
- run.AddMount(cacheDir, cache,
- llb.AsPersistentCacheDir(g.CacheID(cacheDir), llb.CacheMountShared), llb.SourcePath("/cache"))
- return run.Root()
+
+ if len(g.PyPIPackages) != 0 {
+ // Compose the package install command.
+ var sb strings.Builder
+ // Always use the conda's pip.
+ sb.WriteString("/opt/conda/envs/envd/bin/python -m pip install")
+ for _, pkg := range g.PyPIPackages {
+ sb.WriteString(fmt.Sprintf(" %s", pkg))
+ }
+
+ cmd := sb.String()
+ logrus.WithField("command", cmd).
+ Debug("Configure pip install statements")
+ root = llb.User("envd")(root)
+ run := root.
+ Run(llb.Shlex(sb.String()), llb.WithCustomNamef("pip install %s",
+ strings.Join(g.PyPIPackages, " ")))
+ // Refer to https://github.com/moby/buildkit/blob/31054718bf775bf32d1376fe1f3611985f837584/frontend/dockerfile/dockerfile2llb/convert_runmount.go#L46
+ run.AddMount(cacheDir, cache,
+ llb.AsPersistentCacheDir(g.CacheID(cacheDir), llb.CacheMountShared), llb.SourcePath("/cache"))
+ root = run.Root()
+ }
+
+ if g.RequirementsFile != nil {
+ // Compose the package install command.
+ var sb strings.Builder
+ sb.WriteString("/opt/conda/envs/envd/bin/python -m pip install -r ")
+ sb.WriteString(*g.RequirementsFile)
+ cmd := sb.String()
+ logrus.WithField("command", cmd).
+ Debug("Configure pip install requirements statements")
+ root = root.Dir(g.getWorkingDir())
+ run := root.
+ Run(llb.Shlex(cmd), llb.WithCustomNamef("pip install %s",
+ strings.Join(g.PyPIPackages, " ")))
+ run.AddMount(cacheDir, cache,
+ llb.AsPersistentCacheDir(g.CacheID(cacheDir), llb.CacheMountShared), llb.SourcePath("/cache"))
+ run.AddMount(g.getWorkingDir(),
+ llb.Local(flag.FlagBuildContext), llb.Readonly)
+ root = run.Root()
+ }
+ return root
 }
 
 func (g Graph) compilePyPIIndex(root llb.State) llb.State {

pkg/lang/ir/system.go

@@ -55,10 +55,23 @@ func (g Graph) compileRun(root llb.State) llb.State {
  return root.Run(llb.Shlex(fmt.Sprintf("bash -c \"%s\"", g.Exec[0]))).Root()
  }
 
- run := root.Run(llb.Shlex(fmt.Sprintf("bash -c \"%s\"", g.Exec[0])))
- for _, c := range g.Exec[1:] {
- run = run.Run(llb.Shlex(fmt.Sprintf("bash -c \"%s\"", c)))
+ var sb strings.Builder
+ sb.WriteString("set -euo pipefail\n")
+ for _, c := range g.Exec {
+ sb.WriteString(c + "\n")
  }
+
+ cmdStr := fmt.Sprintf("bash -c '%s'", sb.String())
+ logrus.WithField("command", cmdStr).Debug("compile run command")
+ workingDir := g.getWorkingDir()
+ run := root.Dir(workingDir).
+ Run(llb.Shlex(cmdStr))
+ // Mount the build context into the build process.
+ // TODO(gaocegege): Maybe we should make it readonly,
+ // but these cases then cannot be supported:
+ // run(commands=["git clone xx.git"])
+ run.AddMount(workingDir, llb.Local(flag.FlagBuildContext))
+
  return run.Root()
 }
 

pkg/lang/ir/types.go

@@ -43,10 +43,11 @@ type Graph struct {
 
  PublicKeyPath string
 
- PyPIPackages []string
- RPackages []string
- JuliaPackages []string
- SystemPackages []string
+ PyPIPackages []string
+ RequirementsFile *string
+ RPackages []string
+ JuliaPackages []string
+ SystemPackages []string
 
  VSCodePlugins []vscode.Plugin
 
@@ -60,8 +61,11 @@ type Graph struct {
  *CondaConfig
  *RStudioServerConfig
 
- Writer compileui.Writer
- CachePrefix string
+ Writer compileui.Writer
+ // EnvironmentName is the base name of the environment.
+ // It is the BaseDir(BuildContextDir)
+ // e.g. mnist, streamlit-mnist
+ EnvironmentName string
 
  RuntimeGraph
 }

pkg/lang/ir/util.go

@@ -16,12 +16,17 @@ package ir
 
 import (
  "os/user"
+ "path/filepath"
  "regexp"
  "strconv"
 
  "github.com/cockroachdb/errors"
 )
 
+func (g Graph) getWorkingDir() string {
+ return filepath.Join("/home/envd", g.EnvironmentName)
+}
+
 func parseLanguage(l string) (string, *string, error) {
  var language, version string
  if l == "" {