feat: Use UID 1000 to build (#173)

* feat: Use UID 1000 to build

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Fix test cases

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

pkg/docker/docker.go

@@ -173,6 +173,7 @@ func (c generalClient) StartEnvd(ctx context.Context, tag, name, buildContext st
  })
  config := &container.Config{
  Image: tag,
+ User: "envd",
  Entrypoint: []string{
  "tini",
  "--",

pkg/editor/jupyter/util.go

@@ -26,7 +26,7 @@ func GenerateCommand(g ir.Graph, notebookDir string) []string {
  }
 
  cmd := []string{
- "jupyter", "notebook", "--allow-root",
+ "python3", "-m", "notebook",
  "--ip", "0.0.0.0", "--notebook-dir", notebookDir,
  }
  if g.JupyterConfig.Password != "" {

pkg/editor/jupyter/util_test.go

@@ -21,8 +21,7 @@ func TestGenerateCommand(t *testing.T) {
  },
  dir: "test",
  expected: []string{
- "jupyter", "notebook", "--allow-root",
- "--ip", "0.0.0.0", "--notebook-dir", "test",
+ "python3", "-m", "notebook", "--ip", "0.0.0.0", "--notebook-dir", "test",
  "--NotebookApp.password", "''", "--NotebookApp.token", "''",
  "--port", "8888",
  },
@@ -36,8 +35,7 @@ func TestGenerateCommand(t *testing.T) {
  },
  dir: "test",
  expected: []string{
- "jupyter", "notebook", "--allow-root",
- "--ip", "0.0.0.0", "--notebook-dir", "test",
+ "python3", "-m", "notebook", "--ip", "0.0.0.0", "--notebook-dir", "test",
  "--NotebookApp.password", "test", "--NotebookApp.token", "''",
  "--port", "8888",
  },

pkg/lang/ir/compile.go

@@ -0,0 +1,112 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ir
+
+import (
+ "context"
+ "os"
+
+ "github.com/cockroachdb/errors"
+ "github.com/moby/buildkit/client/llb"
+
+ "github.com/tensorchord/envd/pkg/progress/compileui"
+)
+
+func NewGraph() *Graph {
+ return &Graph{
+ OS: osDefault,
+ Language: languageDefault,
+ CUDA: nil,
+ CUDNN: nil,
+ BuiltinSystemPackages: []string{
+ // TODO(gaocegege): Move them into the base image.
+ "curl",
+ "openssh-client",
+ "git",
+ "sudo",
+ "tini",
+ },
+
+ PyPIPackages: []string{},
+ SystemPackages: []string{},
+ Exec: []string{},
+ Shell: shellBASH,
+ }
+}
+
+var DefaultGraph = NewGraph()
+
+func GPUEnabled() bool {
+ return DefaultGraph.CUDA != nil
+}
+
+func Compile(ctx context.Context) (*llb.Definition, error) {
+ w, err := compileui.New(ctx, os.Stdout, "auto")
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to create compileui")
+ }
+ DefaultGraph.Writer = w
+ state, err := DefaultGraph.Compile()
+ if err != nil {
+ return nil, err
+ }
+ // TODO(gaocegege): Support multi platform.
+ def, err := state.Marshal(ctx, llb.LinuxAmd64)
+ if err != nil {
+ return nil, err
+ }
+ return def, nil
+}
+
+func (g Graph) Compile() (llb.State, error) {
+ // TODO(gaocegege): Support more OS and langs.
+ base := g.compileBase()
+ aptStage := g.compileUbuntuAPT(base)
+ pypiMirrorStage := g.compilePyPIMirror(aptStage)
+
+ g.compileJupyter()
+ // TODO(gaocegege): Make apt update a seperate stage to
+ // parallel system and user-defined package installation.
+ builtinSystemStage := g.compileBuiltinSystemPackages(pypiMirrorStage)
+ shellStage, err := g.compileShell(builtinSystemStage)
+ if err != nil {
+ return llb.State{}, errors.Wrap(err, "failed to compile shell")
+ }
+ diffShellStage := llb.Diff(builtinSystemStage, shellStage, llb.WithCustomName("install shell"))
+ pypiStage := llb.Diff(builtinSystemStage, g.compilePyPIPackages(builtinSystemStage), llb.WithCustomName("install PyPI packages"))
+ systemStage := llb.Diff(builtinSystemStage, g.compileSystemPackages(builtinSystemStage), llb.WithCustomName("install system packages"))
+ sshStage := g.copyEnvdSSHServer()
+
+ vscodeStage, err := g.compileVSCode()
+ if err != nil {
+ return llb.State{}, errors.Wrap(err, "failed to get vscode plugins")
+ }
+
+ var merged llb.State
+ if vscodeStage != nil {
+ merged = llb.Merge([]llb.State{
+ builtinSystemStage, systemStage, pypiStage, sshStage, *vscodeStage, diffShellStage,
+ }, llb.WithCustomName("merging all components into one"))
+ } else {
+ merged = llb.Merge([]llb.State{
+ builtinSystemStage, systemStage, pypiStage, sshStage, diffShellStage,
+ }, llb.WithCustomName("merging all components into one"))
+ }
+
+ // TODO(gaocegege): Support order-based exec.
+ run := g.compileRun(merged)
+ g.Writer.Finish()
+ return run, nil
+}

pkg/lang/ir/consts.go

@@ -25,4 +25,6 @@ const (
  pypiConfigTemplate = `
 [global]
 index-url=%s`
+
+ defaultUID = "1000"
 )

pkg/lang/ir/editor.go

@@ -0,0 +1,40 @@
+package ir
+
+import (
+ "github.com/moby/buildkit/client/llb"
+ "github.com/tensorchord/envd/pkg/editor/vscode"
+ "github.com/tensorchord/envd/pkg/flag"
+ "github.com/tensorchord/envd/pkg/progress/compileui"
+)
+
+func (g Graph) compileVSCode() (*llb.State, error) {
+ if len(g.VSCodePlugins) == 0 {
+ return nil, nil
+ }
+ inputs := []llb.State{}
+ for _, p := range g.VSCodePlugins {
+ vscodeClient := vscode.NewClient()
+ g.Writer.LogVSCodePlugin(p, compileui.ActionStart, false)
+ if cached, err := vscodeClient.DownloadOrCache(p); err != nil {
+ return nil, err
+ } else {
+ g.Writer.LogVSCodePlugin(p, compileui.ActionEnd, cached)
+ }
+ ext := llb.Scratch().File(llb.Copy(llb.Local(flag.FlagCacheDir),
+ vscodeClient.PluginPath(p),
+ "/home/envd/.vscode-server/extensions/"+p.String(),
+ &llb.CopyInfo{
+ CreateDestPath: true,
+ }, llb.WithUser(defaultUID)),
+ llb.WithCustomNamef("install vscode plugin %s", p.String()))
+ inputs = append(inputs, ext)
+ }
+ layer := llb.Merge(inputs, llb.WithCustomName("merging plugins for vscode"))
+ return &layer, nil
+}
+
+func (g *Graph) compileJupyter() {
+ if g.JupyterConfig != nil {
+ g.PyPIPackages = append(g.PyPIPackages, "jupyter")
+ }
+}