refact: apt_source, io, config mode (#853)

* refact: apt_source, io, config mode

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* format

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* fix lint

Signed-off-by: Keming <kemingyang@tensorchord.ai>

Signed-off-by: Keming <kemingyang@tensorchord.ai>

e2e/cli/testdata/up-test/build.envd

@@ -3,7 +3,7 @@ def build():
     install.python_packages(name = [
         "via",
     ])
-    install.system_packages(name = ["screenfetch"])
+    install.apt_packages(name = ["screenfetch"])
     shell("zsh")
     config.pip_index(url = "https://pypi.tuna.tsinghua.edu.cn/simple")
     # git_config(name="envd", email="envd@envd", editor="vim")

e2e/docs/testdata/complex/build.envd

@@ -20,7 +20,7 @@ deb https://mirror.sjtu.edu.cn/ubuntu focal-security main restricted universe mu
     ])
     install.cuda(version="11.6", cudnn="8")
     shell("zsh")
-    install.system_packages(name = [
+    install.apt_packages(name = [
         "htop"
     ])
     git_config(name="Ce Gao", email="cegao@tensorchord.ai", editor="vim")

envd/api/config/__init__.py

@@ -24,7 +24,7 @@
 from typing import Optional, List
 
 
-def apt_source(mode: Optional[str], source: Optional[str]):
+def apt_source(source: Optional[str]):
     """Configure apt sources
 
     Example usage:
@@ -43,7 +43,6 @@ def apt_source(mode: Optional[str], source: Optional[str]):
     ```
 
     Args:
-        mode (str, optional): This argument is not supported currently
         source (str, optional): The apt source configuration
     """
 
@@ -57,11 +56,10 @@ def jupyter(token: str, port: int):
     """
 
 
-def pip_index(mode: str, url: str, extra_url: str):
+def pip_index(url: str, extra_url: str):
     """Configure pypi index mirror
 
     Args:
-        mode (str): NOT supported yet
         url (str): PyPI index URL (i.e. https://mirror.sjtu.edu.cn/pypi/web/simple)
         extra_url (str): PyPI extra index URL. `url` and `extra_url` will be
             treated equally, see https://github.com/pypa/pip/issues/8606

envd/api/install/__init__.py

@@ -24,8 +24,8 @@
 from typing import List, Optional
 
 
-def system_packages(name: List[str]):
-    """Install package by system-level package manager(apt on Ubuntu)
+def apt_packages(name: List[str]):
+    """Install package by system-level package manager (apt on Ubuntu)
 
     Args:
         name (str): apt package name list

envd/api/io/__init__.py

@@ -22,19 +22,19 @@
 """
 
 
-def copy(src: str, dest: str):
-    """Copy from host `src` to container `dest` (build time)
+def copy(host_path: str, envd_path: str):
+    """Copy from host path to container path (build time)
 
     Args:
-        src (str): source path
-        dest (str): destination path
+        host_path (str): source path in the host machine
+        envd_path (str): destination path in the envd container
     """
 
 
-def mount(src: str, dest: str):
-    """Mount from host `src` to container `dest` (runtime)
+def mount(host_path: str, envd_path: str):
+    """Mount from host path to container path (runtime)
 
     Args:
-        src (str): source path
-        dest (str): destination path
+        host_path (str): source path in the host machine
+        envd_path (str): destination path in the envd container
     """

examples/dgl/build.envd

@@ -8,8 +8,8 @@ def build():
     # Select the shell environment you like
     shell("zsh")
 
-    # io.mount(src="~/.envd/data/dgl", dest="~/.dgl")
-    io.mount(src=data.envd("dgl"), dest=data.path.dgl)
+    # io.mount(host_path="~/.envd/data/dgl", envd_path="~/.dgl")
+    io.mount(host_path=data.envd("dgl"), envd_path=data.path.dgl)
 
 def build_gpu():
     # Use ubuntu20.04 as base image and install python    
@@ -26,4 +26,4 @@ def build_gpu():
     # Select the shell environment you like
     shell("zsh")
 
-    io.mount(src=data.envd("dgl"), dest=data.path.dgl)
+    io.mount(host_path=data.envd("dgl"), envd_path=data.path.dgl)

examples/ianvs/build.envd

@@ -1,7 +1,7 @@
 def build():
     base(os="ubuntu20.04", language="python3.6")
     shell("zsh")
-    install.system_packages(name=["git", "libgl1-mesa-glx", "zip"])
+    install.apt_packages(name=["git", "libgl1-mesa-glx", "zip"])
     run(commands=[
       "git clone https://github.com/kubeedge/ianvs.git",
       "cd ./ianvs",

examples/python-basic/build.envd

@@ -11,7 +11,7 @@ def build():
     install.python_packages([
         "via",
     ])
-    io.copy(src="./build.envd", dest="/")
+    io.copy(host_path="./build.envd", envd_path="/")
     runtime.command(commands={
         "test": "ls /",
     })

examples/streamlit-mnist/build.envd

@@ -27,7 +27,7 @@ def configure_streamlit(port):
 
 def configure_mnist():
     # config.pip_index(url = "https://pypi.tuna.tsinghua.edu.cn/simple")
-    install.system_packages([
+    install.apt_packages([
         "libgl1",
     ])
     install.python_packages([

pkg/lang/frontend/starlark/config/config.go

@@ -92,20 +92,19 @@ func ruleFuncJupyter(thread *starlark.Thread, _ *starlark.Builtin,
 
 func ruleFuncPyPIIndex(thread *starlark.Thread, _ *starlark.Builtin,
 	args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
-	var mode, url, extraURL starlark.String
+	var url, extraURL starlark.String
 
 	if err := starlark.UnpackArgs(rulePyPIIndex, args, kwargs,
-		"mode?", &mode, "url?", &url, "extra_url?", &extraURL); err != nil {
+		"url?", &url, "extra_url?", &extraURL); err != nil {
 		return nil, err
 	}
 
-	modeStr := mode.GoString()
 	indexStr := url.GoString()
 	extraIndexStr := extraURL.GoString()
 
-	logger.Debugf("rule `%s` is invoked, mode=%s, index=%s, extraIndex=%s",
-		rulePyPIIndex, modeStr, indexStr, extraIndexStr)
-	if err := ir.PyPIIndex(modeStr, indexStr, extraIndexStr); err != nil {
+	logger.Debugf("rule `%s` is invoked, index=%s, extraIndex=%s",
+		rulePyPIIndex, indexStr, extraIndexStr)
+	if err := ir.PyPIIndex(indexStr, extraIndexStr); err != nil {
 		return nil, err
 	}
 
@@ -150,19 +149,17 @@ func ruleFuncJuliaPackageServer(thread *starlark.Thread, _ *starlark.Builtin,
 
 func ruleFuncUbuntuAptSource(thread *starlark.Thread, _ *starlark.Builtin,
 	args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
-	var mode, source starlark.String
+	var source starlark.String
 
 	if err := starlark.UnpackArgs(ruleUbuntuAptSource, args, kwargs,
-		"mode?", &mode, "source?", &source); err != nil {
+		"source?", &source); err != nil {
 		return nil, err
 	}
 
-	modeStr := mode.GoString()
 	sourceStr := source.GoString()
 
-	logger.Debugf("rule `%s` is invoked, mode=%s, source=%s",
-		ruleUbuntuAptSource, modeStr, sourceStr)
-	if err := ir.UbuntuAPT(modeStr, sourceStr); err != nil {
+	logger.Debugf("rule `%s` is invoked, source=%s", ruleUbuntuAptSource, sourceStr)
+	if err := ir.UbuntuAPT(sourceStr); err != nil {
 		return nil, err
 	}
 

pkg/lang/frontend/starlark/install/const.go

@@ -15,7 +15,7 @@
 package install
 
 const (
-	ruleSystemPackage = "install.system_packages"
+	ruleSystemPackage = "install.apt_packages"
 	rulePyPIPackage   = "install.python_packages"
 	ruleRPackage      = "install.r_packages"
 	ruleCUDA          = "install.cuda"

pkg/lang/frontend/starlark/install/install.go

@@ -34,7 +34,7 @@ var Module = &starlarkstruct.Module{
 	Members: starlark.StringDict{
 		"python_packages":   starlark.NewBuiltin(rulePyPIPackage, ruleFuncPyPIPackage),
 		"r_packages":        starlark.NewBuiltin(ruleRPackage, ruleFuncRPackage),
-		"system_packages":   starlark.NewBuiltin(ruleSystemPackage, ruleFuncSystemPackage),
+		"apt_packages":      starlark.NewBuiltin(ruleSystemPackage, ruleFuncSystemPackage),
 		"cuda":              starlark.NewBuiltin(ruleCUDA, ruleFuncCUDA),
 		"vscode_extensions": starlark.NewBuiltin(ruleVSCode, ruleFuncVSCode),
 		"conda_packages":    starlark.NewBuiltin(ruleConda, ruleFuncConda),

pkg/lang/frontend/starlark/io/io.go

@@ -47,7 +47,7 @@ func ruleFuncMount(thread *starlark.Thread, _ *starlark.Builtin,
 	var destination starlark.String
 
 	if err := starlark.UnpackArgs(ruleMount, args, kwargs,
-		"src?", &source, "dest?", &destination); err != nil {
+		"host_path?", &source, "envd_path?", &destination); err != nil {
 		return nil, err
 	}
 
@@ -99,7 +99,7 @@ func ruleFuncCopy(thread *starlark.Thread, _ *starlark.Builtin,
 	var source, destination starlark.String
 
 	if err := starlark.UnpackArgs(ruleCopy, args, kwargs,
-		"src?", &source, "dest?", &destination); err != nil {
+		"host_path?", &source, "envd_path?", &destination); err != nil {
 		return nil, err
 	}
 

pkg/lang/ir/consts.go

@@ -18,7 +18,6 @@ const (
 	osDefault              = "ubuntu20.04"
 	languageDefault        = "python"
 	languageVersionDefault = "3"
-	pypiIndexModeAuto      = "auto"
 
 	// used inside the container
 	defaultConfigDir   = "/home/envd/.config"

pkg/lang/ir/interface.go

@@ -81,27 +81,17 @@ func VSCodePlugins(plugins []string) error {
 }
 
 // UbuntuAPT updates the Ubuntu apt source.list in the image.
-func UbuntuAPT(mode, source string) error {
+func UbuntuAPT(source string) error {
 	if source == "" {
-		if mode == pypiIndexModeAuto {
-			// If the mode is set to `auto`, envd detects the location of the run
-			// then set to the nearest mirror
-			return errors.New("auto-mode not implemented")
-		}
 		return errors.New("source is required")
 	}
 
 	DefaultGraph.UbuntuAPTSource = &source
 	return nil
 }
 
-func PyPIIndex(mode, url, extraURL string) error {
+func PyPIIndex(url, extraURL string) error {
 	if url == "" {
-		if mode == pypiIndexModeAuto {
-			// If the mode is set to `auto`, envd detects the location of the run
-			// then set to the nearest index URL.
-			return errors.New("auto-mode not implemented")
-		}
 		return errors.New("url is required")
 	}