test: add e2e test for examples in documentations (#804)

* add

Signed-off-by: Jinjing.Zhou <allenzhou@tensorchord.ai>

* fix

Signed-off-by: Jinjing.Zhou <allenzhou@tensorchord.ai>

* lint

Signed-off-by: Jinjing.Zhou <allenzhou@tensorchord.ai>

* add

Signed-off-by: Jinjing.Zhou <allenzhou@tensorchord.ai>

* add

Signed-off-by: Jinjing.Zhou <allenzhou@tensorchord.ai>

* remove blank line

Signed-off-by: Jinjing.Zhou <allenzhou@tensorchord.ai>

Signed-off-by: Jinjing.Zhou <allenzhou@tensorchord.ai>

Makefile

@@ -181,6 +181,16 @@ e2e-test:
 		-X $(ROOT)/pkg/version.developmentFlag=true" \
 		-race -v -timeout 20m -coverpkg=./pkg/app -coverprofile=e2e-coverage.out ./e2e
 
+doc-test:
+	@go test -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) \
+		-X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) \
+		-X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) \
+		-X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)                     \
+		-X $(ROOT)/pkg/version.gitTag="$(shell git describe --tags --abbrev=0)" \
+		-X $(ROOT)/pkg/version.developmentFlag=true" \
+		-race -v -timeout 20m -coverpkg=./pkg/app -coverprofile=e2e-coverage.out ./e2e/docs
+
+
 clean:  ## Clean the outputs and artifacts
 	@-rm -vrf ${OUTPUT_DIR}
 	@-rm -vrf ${DEBUG_DIR}

e2e/docs/docs_test.go

@@ -0,0 +1,95 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docs
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/tensorchord/envd/e2e"
+	"github.com/tensorchord/envd/pkg/app"
+	"github.com/tensorchord/envd/pkg/home"
+)
+
+var _ = Describe("check examples in documentation", func() {
+	buildContext := "doctest"
+	// env := "up-test"
+	baseArgs := []string{
+		"envd.test", "--debug",
+	}
+
+	BeforeEach(func() {
+		Expect(home.Initialize()).NotTo(HaveOccurred())
+		e2e.ResetEnvdApp()
+		envdApp := app.New()
+		err := envdApp.Run(append(baseArgs, "bootstrap"))
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("can list envd envs", func() {
+		envdApp := app.New()
+		err := envdApp.Run([]string{"envd.test", "--debug", "envs", "list"})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("can check envd envs details", func() {
+		buildContext := "testdata/minimal"
+		args := append(baseArgs, []string{
+			"up", "--path", buildContext, "--detach", "--force",
+		}...)
+		e2e.ResetEnvdApp()
+		envdApp := app.New()
+		err := envdApp.Run(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = envdApp.Run([]string{"envd.test", "--debug", "envs", "describe"})
+		Expect(err).NotTo(HaveOccurred())
+
+		destroyArgs := append(baseArgs, []string{
+			"destroy", "--path", buildContext,
+		}...)
+		err = envdApp.Run(destroyArgs)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	up_tests := []string{"testdata/minimal", "testdata/getting_started", "testdata/jupyter", "testsdata/complex"}
+
+	for _, v := range up_tests {
+		It(fmt.Sprintf("can up %s environment", v), func() {
+			args := append(baseArgs, []string{
+				"up", "--path", "testdata/getting_started", "-f", "build.envd", "--detach", "--force",
+			}...)
+			e2e.ResetEnvdApp()
+			envdApp := app.New()
+			err := envdApp.Run(args)
+			Expect(err).NotTo(HaveOccurred())
+
+			destroyArgs := append(baseArgs, []string{
+				"destroy", "--path", buildContext,
+			}...)
+			err = envdApp.Run(destroyArgs)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	}
+
+})
+
+func TestMain(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "envd documentation example test suite")
+}

e2e/docs/testdata/complex/build.envd

@@ -0,0 +1,27 @@
+def build():
+    config.apt_source(source="""
+deb https://mirror.sjtu.edu.cn/ubuntu focal main restricted
+deb https://mirror.sjtu.edu.cn/ubuntu focal-updates main restricted
+deb https://mirror.sjtu.edu.cn/ubuntu focal universe
+deb https://mirror.sjtu.edu.cn/ubuntu focal-updates universe
+deb https://mirror.sjtu.edu.cn/ubuntu focal multiverse
+deb https://mirror.sjtu.edu.cn/ubuntu focal-updates multiverse
+deb https://mirror.sjtu.edu.cn/ubuntu focal-backports main restricted universe multiverse
+deb http://archive.canonical.com/ubuntu focal partner
+deb https://mirror.sjtu.edu.cn/ubuntu focal-security main restricted universe multiverse
+""")
+    config.pip_index(url = "https://mirror.sjtu.edu.cn/pypi/web/simple")
+    install.vscode_extensions([
+        "ms-python.python",
+    ])
+    base(os="ubuntu20.04", language="python3")
+    install.python_packages(name = [
+        "numpy",
+    ])
+    install.cuda(version="11.6", cudnn="8")
+    shell("zsh")
+    install.system_packages(name = [
+        "htop"
+    ])
+    git_config(name="Ce Gao", email="cegao@tensorchord.ai", editor="vim")
+    run(["ls -la"])

e2e/docs/testdata/getting_started/build.envd

@@ -0,0 +1,8 @@
+def build():
+    base(os="ubuntu20.04", language="python3")
+    # Configure pip index if needed.
+    #config.pip_index(url = "https://pypi.tuna.tsinghua.edu.cn/simple")
+    install.python_packages(name = [
+        "numpy",
+    ])
+    shell("zsh")

e2e/docs/testdata/jupyter/build.envd

@@ -0,0 +1,7 @@
+def build():
+    base(os="ubuntu20.04", language="python3")
+    install.python_packages(name = [
+        "numpy",
+    ])
+    shell("zsh")
+    config.jupyter(token="")

e2e/docs/testdata/minimal/build.envd

@@ -0,0 +1,5 @@
+def build():
+    base(os="ubuntu20.04", language="python3")
+    install.python_packages(name = [
+        "via",
+    ])

e2e/testdata/quick-start/build.envd

@@ -3,4 +3,4 @@ def build():
     install.python_packages(name = [
         "numpy",
     ])
-    shell("zsh")
+    shell("zsh")