feat: Refactor syntax (#238)

* add

* refactor syntax

* refactor

* fix

* remove comment

* remove lib folder

* remove graph

* add

* address suggestion

* use full name

* fix

README.md

@@ -60,18 +60,18 @@ Checkout the [examples](./examples/mnist), and configure envd with the manifest
 
 ```python
 def build():
- vscode(plugins = [
+ install.vscode_extensions([
  "ms-python.python",
  ])
 
  base(os="ubuntu20.04", language="python3")
- pip_package(name = [
+ install.python_packages(name = [
  "tensorflow",
  "numpy",
  ])
- cuda(version="11.6", cudnn="8")
+ install.cuda(version="11.6", cudnn="8")
  shell("zsh")
- jupyter(password="", port=8888)
+ config.jupyter(password="", port=8888)
 ```
 
 Then you can run `envd up` to create the development environment.

cmd/envd/testdata/build.envd

@@ -1,6 +1,6 @@
 def build():
  base(os="ubuntu20.04", language="python3")
- pip_package(name = [
+ install.python_packages(name = [
  "ormb",
  ])
- install_package(name = ["screenfetch"])
+ install.system_packages(name = ["screenfetch"])

examples/mnist/build.envd

@@ -1,13 +1,13 @@
 def build():
- vscode(plugins = [
+ base(os="ubuntu20.04", language="python3")
+ install.vscode_extensions([
  "ms-python.python",
  ])
 
- base(os="ubuntu20.04", language="python3")
- pip_package(name = [
+ install.python_packages([
  "tensorflow",
  "numpy",
  ])
- cuda(version="11.6", cudnn="8")
+ install.cuda(version="11.6", cudnn="8")
  shell("zsh")
- jupyter(password="", port=8888)
+ config.jupyter(password="", port=8888)

pkg/lang/frontend/starlark/config/config.go

@@ -0,0 +1,123 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+ "github.com/cockroachdb/errors"
+ "github.com/sirupsen/logrus"
+ "github.com/tensorchord/envd/pkg/lang/ir"
+ "go.starlark.net/starlark"
+ "go.starlark.net/starlarkstruct"
+)
+
+var (
+ logger = logrus.WithField("frontend", "starlark")
+)
+
+var Module = &starlarkstruct.Module{
+ Name: "config",
+ Members: starlark.StringDict{
+ "apt_source": starlark.NewBuiltin(ruleUbuntuAptSource, ruleFuncUbuntuAptSource),
+ "jupyter": starlark.NewBuiltin(ruleJupyter, ruleFuncJupyter),
+ "pip_index": starlark.NewBuiltin(
+ rulePyPIIndex, ruleFuncPyPIIndex),
+ },
+}
+
+func ruleFuncJupyter(thread *starlark.Thread, _ *starlark.Builtin,
+ args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var password starlark.String
+ var port starlark.Int
+
+ if err := starlark.UnpackArgs(ruleJupyter, args, kwargs,
+ "password?", &password, "port?", &port); err != nil {
+ return nil, err
+ }
+
+ pwdStr := ""
+ if password != starlark.String("") {
+ pwdStr = password.GoString()
+ }
+
+ portInt, ok := port.Int64()
+ if !ok {
+ return nil, errors.New("port must be an integer")
+ }
+ logger.Debugf("rule `%s` is invoked, password=%s, port=%d", ruleJupyter,
+ pwdStr, portInt)
+ if err := ir.Jupyter(pwdStr, portInt); err != nil {
+ return nil, err
+ }
+
+ return starlark.None, nil
+}
+
+func ruleFuncPyPIIndex(thread *starlark.Thread, _ *starlark.Builtin,
+ args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var mode, url, extraURL starlark.String
+
+ if err := starlark.UnpackArgs(rulePyPIIndex, args, kwargs,
+ "mode?", &mode, "url?", &url, "extra_url?", &extraURL); err != nil {
+ return nil, err
+ }
+
+ modeStr := ""
+ if mode != starlark.String("") {
+ modeStr = mode.GoString()
+ }
+ indexStr := ""
+ if url != starlark.String("") {
+ indexStr = url.GoString()
+ }
+ extraIndexStr := ""
+ if extraURL != starlark.String("") {
+ extraIndexStr = extraURL.GoString()
+ }
+
+ logger.Debugf("rule `%s` is invoked, mode=%s, index=%s, extraIndex=%s", rulePyPIIndex,
+ modeStr, indexStr, extraIndexStr)
+ if err := ir.PyPIIndex(modeStr, indexStr, extraIndexStr); err != nil {
+ return nil, err
+ }
+
+ return starlark.None, nil
+}
+
+func ruleFuncUbuntuAptSource(thread *starlark.Thread, _ *starlark.Builtin,
+ args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var mode, source starlark.String
+
+ if err := starlark.UnpackArgs(ruleUbuntuAptSource, args, kwargs,
+ "mode?", &mode, "source?", &source); err != nil {
+ return nil, err
+ }
+
+ modeStr := ""
+ if mode != starlark.String("") {
+ modeStr = mode.GoString()
+ }
+ sourceStr := ""
+ if source != starlark.String("") {
+ sourceStr = source.GoString()
+ }
+
+ logger.Debugf("rule `%s` is invoked, mode=%s, source=%s", ruleUbuntuAptSource,
+ modeStr, sourceStr)
+ if err := ir.UbuntuAPT(modeStr, sourceStr); err != nil {
+ return nil, err
+ }
+
+ return starlark.None, nil
+}

pkg/lang/frontend/starlark/const.go → pkg/lang/frontend/starlark/config/const.go

@@ -12,18 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package starlark
+package config
 
 const (
- ruleBase = "base"
- ruleSystemPackage = "install_package"
- rulePyPIPackage = "pip_package"
- ruleCUDA = "cuda"
- ruleVSCode = "vscode"
- ruleUbuntuAPT = "ubuntu_apt"
- rulePyPIIndex = "pip_index"
- ruleShell = "shell"
- ruleJupyter = "jupyter"
- ruleRun = "run"
- ruleGitConfig = "git_config"
+ ruleUbuntuAptSource = "config.apt_source"
+ rulePyPIIndex = "config.pip_index"
+ ruleJupyter = "config.jupyter"
 )

pkg/lang/frontend/starlark/install/const.go

@@ -0,0 +1,22 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package install
+
+const (
+ ruleSystemPackage = "install.system_packages"
+ rulePyPIPackage = "install.python_packages"
+ ruleCUDA = "install.cuda"
+ ruleVSCode = "install.vscode_extensions"
+)

pkg/lang/frontend/starlark/install/install.go

@@ -0,0 +1,132 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package install
+
+import (
+ "github.com/sirupsen/logrus"
+ "github.com/tensorchord/envd/pkg/lang/ir"
+ "go.starlark.net/starlark"
+ "go.starlark.net/starlarkstruct"
+)
+
+var (
+ logger = logrus.WithField("frontend", "starlark")
+)
+
+var Module = &starlarkstruct.Module{
+ Name: "install",
+ Members: starlark.StringDict{
+ "python_packages": starlark.NewBuiltin(
+ rulePyPIPackage, ruleFuncPyPIPackage),
+ "system_packages": starlark.NewBuiltin(
+ ruleSystemPackage, ruleFuncSystemPackage),
+ "cuda": starlark.NewBuiltin(ruleCUDA, ruleFuncCUDA),
+ "vscode_extensions": starlark.NewBuiltin(ruleVSCode, ruleFuncVSCode),
+ "shell": starlark.NewBuiltin(ruleVSCode, ruleFuncVSCode),
+ },
+}
+
+func ruleFuncPyPIPackage(thread *starlark.Thread, _ *starlark.Builtin,
+ args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var name *starlark.List
+
+ if err := starlark.UnpackArgs(rulePyPIPackage,
+ args, kwargs, "name", &name); err != nil {
+ return nil, err
+ }
+
+ nameList := []string{}
+ if name != nil {
+ for i := 0; i < name.Len(); i++ {
+ nameList = append(nameList, name.Index(i).(starlark.String).GoString())
+ }
+ }
+
+ logger.Debugf("rule `%s` is invoked, name=%v", rulePyPIPackage, nameList)
+ ir.PyPIPackage(nameList)
+
+ return starlark.None, nil
+}
+
+func ruleFuncSystemPackage(thread *starlark.Thread, _ *starlark.Builtin,
+ args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var name *starlark.List
+
+ if err := starlark.UnpackArgs(ruleSystemPackage,
+ args, kwargs, "name?", &name); err != nil {
+ return nil, err
+ }
+
+ nameList := []string{}
+ if name != nil {
+ for i := 0; i < name.Len(); i++ {
+ nameList = append(nameList, name.Index(i).(starlark.String).GoString())
+ }
+ }
+
+ logger.Debugf("rule `%s` is invoked, name=%v", ruleSystemPackage, nameList)
+ ir.SystemPackage(nameList)
+
+ return starlark.None, nil
+}
+
+func ruleFuncCUDA(thread *starlark.Thread, _ *starlark.Builtin,
+ args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var version, cudnn starlark.String
+
+ if err := starlark.UnpackArgs(ruleCUDA, args, kwargs,
+ "version?", &version, "cudnn?", &cudnn); err != nil {
+ return nil, err
+ }
+
+ versionStr := ""
+ if version != starlark.String("") {
+ versionStr = version.GoString()
+ }
+ cudnnStr := ""
+ if cudnn != starlark.String("") {
+ cudnnStr = cudnn.GoString()
+ }
+
+ logger.Debugf("rule `%s` is invoked, version=%s, cudnn=%s", ruleCUDA,
+ versionStr, cudnnStr)
+ ir.CUDA(versionStr, cudnnStr)
+
+ return starlark.None, nil
+}
+
+func ruleFuncVSCode(thread *starlark.Thread, _ *starlark.Builtin,
+ args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var plugins *starlark.List
+
+ if err := starlark.UnpackArgs(ruleVSCode,
+ args, kwargs, "name", &plugins); err != nil {
+ return nil, err
+ }
+
+ pluginList := []string{}
+ if plugins != nil {
+ for i := 0; i < plugins.Len(); i++ {
+ pluginList = append(pluginList, plugins.Index(i).(starlark.String).GoString())
+ }
+ }
+
+ logger.Debugf("rule `%s` is invoked, plugins=%v", ruleVSCode, pluginList)
+ if err := ir.VSCodePlugins(pluginList); err != nil {
+ return starlark.None, err
+ }
+
+ return starlark.None, nil
+}