feat: Use UID 1000 to build

pkg/docker/docker.go

@@ -173,6 +173,7 @@ func (c generalClient) StartEnvd(ctx context.Context, tag, name, buildContext st
 	})
 	config := &container.Config{
 		Image: tag,
+		User:  "envd",
 		Entrypoint: []string{
 			"tini",
 			"--",

pkg/editor/jupyter/util.go

@@ -26,7 +26,7 @@ func GenerateCommand(g ir.Graph, notebookDir string) []string {
 	}
 
 	cmd := []string{
-		"jupyter", "notebook", "--allow-root",
+		"python3", "-m", "notebook",
 		"--ip", "0.0.0.0", "--notebook-dir", notebookDir,
 	}
 	if g.JupyterConfig.Password != "" {

pkg/editor/jupyter/util_test.go

@@ -21,8 +21,7 @@ func TestGenerateCommand(t *testing.T) {
 			},
 			dir: "test",
 			expected: []string{
-				"jupyter", "notebook", "--allow-root",
-				"--ip", "0.0.0.0", "--notebook-dir", "test",
+				"python3", "-m", "notebook", "--ip", "0.0.0.0", "--notebook-dir", "test",
 				"--NotebookApp.password", "''", "--NotebookApp.token", "''",
 				"--port", "8888",
 			},
@@ -36,8 +35,7 @@ func TestGenerateCommand(t *testing.T) {
 			},
 			dir: "test",
 			expected: []string{
-				"jupyter", "notebook", "--allow-root",
-				"--ip", "0.0.0.0", "--notebook-dir", "test",
+				"python3", "-m", "notebook", "--ip", "0.0.0.0", "--notebook-dir", "test",
 				"--NotebookApp.password", "test", "--NotebookApp.token", "''",
 				"--port", "8888",
 			},

pkg/lang/ir/compile.go

@@ -0,0 +1,112 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ir
+
+import (
+	"context"
+	"os"
+
+	"github.com/cockroachdb/errors"
+	"github.com/moby/buildkit/client/llb"
+
+	"github.com/tensorchord/envd/pkg/progress/compileui"
+)
+
+func NewGraph() *Graph {
+	return &Graph{
+		OS:       osDefault,
+		Language: languageDefault,
+		CUDA:     nil,
+		CUDNN:    nil,
+		BuiltinSystemPackages: []string{
+			// TODO(gaocegege): Move them into the base image.
+			"curl",
+			"openssh-client",
+			"git",
+			"sudo",
+			"tini",
+		},
+
+		PyPIPackages:   []string{},
+		SystemPackages: []string{},
+		Exec:           []string{},
+		Shell:          shellBASH,
+	}
+}
+
+var DefaultGraph = NewGraph()
+
+func GPUEnabled() bool {
+	return DefaultGraph.CUDA != nil
+}
+
+func Compile(ctx context.Context) (*llb.Definition, error) {
+	w, err := compileui.New(ctx, os.Stdout, "auto")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create compileui")
+	}
+	DefaultGraph.Writer = w
+	state, err := DefaultGraph.Compile()
+	if err != nil {
+		return nil, err
+	}
+	// TODO(gaocegege): Support multi platform.
+	def, err := state.Marshal(ctx, llb.LinuxAmd64)
+	if err != nil {
+		return nil, err
+	}
+	return def, nil
+}
+
+func (g Graph) Compile() (llb.State, error) {
+	// TODO(gaocegege): Support more OS and langs.
+	base := g.compileBase()
+	aptStage := g.compileUbuntuAPT(base)
+	pypiMirrorStage := g.compilePyPIMirror(aptStage)
+
+	g.compileJupyter()
+	// TODO(gaocegege): Make apt update a seperate stage to
+	// parallel system and user-defined package installation.
+	builtinSystemStage := g.compileBuiltinSystemPackages(pypiMirrorStage)
+	shellStage, err := g.compileShell(builtinSystemStage)
+	if err != nil {
+		return llb.State{}, errors.Wrap(err, "failed to compile shell")
+	}
+	diffShellStage := llb.Diff(builtinSystemStage, shellStage, llb.WithCustomName("install shell"))
+	pypiStage := llb.Diff(builtinSystemStage, g.compilePyPIPackages(builtinSystemStage), llb.WithCustomName("install PyPI packages"))
+	systemStage := llb.Diff(builtinSystemStage, g.compileSystemPackages(builtinSystemStage), llb.WithCustomName("install system packages"))
+	sshStage := g.copyEnvdSSHServer()
+
+	vscodeStage, err := g.compileVSCode()
+	if err != nil {
+		return llb.State{}, errors.Wrap(err, "failed to get vscode plugins")
+	}
+
+	var merged llb.State
+	if vscodeStage != nil {
+		merged = llb.Merge([]llb.State{
+			builtinSystemStage, systemStage, pypiStage, sshStage, *vscodeStage, diffShellStage,
+		}, llb.WithCustomName("merging all components into one"))
+	} else {
+		merged = llb.Merge([]llb.State{
+			builtinSystemStage, systemStage, pypiStage, sshStage, diffShellStage,
+		}, llb.WithCustomName("merging all components into one"))
+	}
+
+	// TODO(gaocegege): Support order-based exec.
+	run := g.compileRun(merged)
+	g.Writer.Finish()
+	return run, nil
+}

pkg/lang/ir/consts.go

@@ -25,4 +25,6 @@ const (
 	pypiConfigTemplate = `
 [global]
 index-url=%s`
+
+	defaultUID = "1000"
 )

pkg/lang/ir/editor.go

@@ -0,0 +1,40 @@
+package ir
+
+import (
+	"github.com/moby/buildkit/client/llb"
+	"github.com/tensorchord/envd/pkg/editor/vscode"
+	"github.com/tensorchord/envd/pkg/flag"
+	"github.com/tensorchord/envd/pkg/progress/compileui"
+)
+
+func (g Graph) compileVSCode() (*llb.State, error) {
+	if len(g.VSCodePlugins) == 0 {
+		return nil, nil
+	}
+	inputs := []llb.State{}
+	for _, p := range g.VSCodePlugins {
+		vscodeClient := vscode.NewClient()
+		g.Writer.LogVSCodePlugin(p, compileui.ActionStart, false)
+		if cached, err := vscodeClient.DownloadOrCache(p); err != nil {
+			return nil, err
+		} else {
+			g.Writer.LogVSCodePlugin(p, compileui.ActionEnd, cached)
+		}
+		ext := llb.Scratch().File(llb.Copy(llb.Local(flag.FlagCacheDir),
+			vscodeClient.PluginPath(p),
+			"/home/envd/.vscode-server/extensions/"+p.String(),
+			&llb.CopyInfo{
+				CreateDestPath: true,
+			}, llb.WithUser(defaultUID)),
+			llb.WithCustomNamef("install vscode plugin %s", p.String()))
+		inputs = append(inputs, ext)
+	}
+	layer := llb.Merge(inputs, llb.WithCustomName("merging plugins for vscode"))
+	return &layer, nil
+}
+
+func (g *Graph) compileJupyter() {
+	if g.JupyterConfig != nil {
+		g.PyPIPackages = append(g.PyPIPackages, "jupyter")
+	}
+}