fix: e2e doc test (#926)

* fix: e2e doc test

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* fix destroy

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* fix cuda doc test

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* destroy built img

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* format

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* change doc test to daily

Signed-off-by: Keming <kemingyang@tensorchord.ai>

Signed-off-by: Keming <kemingyang@tensorchord.ai>

.github/workflows/CI.yml

@@ -124,39 +124,6 @@ jobs:
         with:
           name: e2e-cli-coverage-out
           path: e2e-cli-coverage.out
-  e2e-doc:
-    name: e2e-doc
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v3
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.18
-      - name: Cache Go modules
-        uses: actions/cache@preview
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.OS }}-build-${{ env.cache-name }}-
-            ${{ runner.OS }}-build-
-            ${{ runner.OS }}-
-      - uses: actions-ecosystem/action-get-latest-tag@v1
-        id: get-latest-tag
-      - name: e2e doc test
-        run: make e2e-doc-test
-        env:
-          GIT_LATEST_TAG: ${{ steps.get-latest-tag.outputs.tag }}
-      - name: Upload coverage report
-        uses: actions/upload-artifact@v3
-        with:
-          name: e2e-doc-coverage-out
-          path: e2e-doc-coverage.out
   e2e-lang:
     name: e2e-lang
     strategy:

.github/workflows/daily.yml

@@ -0,0 +1,44 @@
+name: daily
+
+on:
+  schedule:
+    - cron: '0 2 * * *' # UTC 2:00 every day
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-doc:
+    name: e2e-doc
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - name: Cache Go modules
+        uses: actions/cache@preview
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.OS }}-build-${{ env.cache-name }}-
+            ${{ runner.OS }}-build-
+            ${{ runner.OS }}-
+      - uses: actions-ecosystem/action-get-latest-tag@v1
+        id: get-latest-tag
+      - name: e2e doc test
+        run: make e2e-doc-test
+        env:
+          GIT_LATEST_TAG: ${{ steps.get-latest-tag.outputs.tag }}
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v3
+        with:
+          name: e2e-doc-coverage-out
+          path: e2e-doc-coverage.out

Makefile

@@ -207,7 +207,7 @@ e2e-doc-test:
 		-X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)                     \
 		-X $(ROOT)/pkg/version.gitTag="$(shell git describe --tags --abbrev=0)" \
 		-X $(ROOT)/pkg/version.developmentFlag=true" \
-		-race -v -timeout 20m -coverpkg=./pkg/app -coverprofile=e2e-doc-coverage.out ./e2e/docs
+		-race -v -timeout 60m -coverpkg=./pkg/app -coverprofile=e2e-doc-coverage.out ./e2e/docs
 
 
 clean:  ## Clean the outputs and artifacts

e2e/cli/bytecode_hash_test.go

@@ -43,7 +43,7 @@ var _ = Describe("bytecode hash cache target", func() {
 	exampleName := "quick-start"
 	It("add some blank to build.envd", func() {
 		testcase := "add-blank"
-		e := e2e.NewExample(buildContextDirWithName(exampleName), testcase)
+		e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 		ctx := context.TODO()
 		e.BuildImage(false)()
 		dockerClient := e2e.GetDockerClient(ctx)

e2e/cli/quick_start_test.go

@@ -24,7 +24,7 @@ import (
 var _ = Describe("e2e quickstart", Ordered, func() {
 	exampleName := "quick-start"
 	testcase := "e2e"
-	e := e2e.NewExample(buildContextDirWithName(exampleName), testcase)
+	e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 	BeforeAll(e.BuildImage(true))
 	BeforeEach(e.RunContainer())
 	It("execute python demo.py", func() {

e2e/cli/suite_test.go

@@ -16,7 +16,6 @@ package cli
 
 import (
 	"os"
-	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -25,11 +24,6 @@ import (
 	"github.com/tensorchord/envd/pkg/version"
 )
 
-func buildContextDirWithName(name string) string {
-	basePath := "testdata"
-	return filepath.Join(basePath, name)
-}
-
 func init() {
 	version.SetGitTagForE2ETest(os.Getenv("GIT_LATEST_TAG"))
 }

e2e/docs/docs_gpu_test.go

@@ -0,0 +1,27 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docs
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+
+	"github.com/tensorchord/envd/e2e"
+)
+
+var _ = Describe("check GPU examples in documentation", Ordered, func() {
+	e := e2e.NewExample(e2e.BuildContextDirWithName("complex"), "e2e-doc")
+	It("should be able to build the GPU example", e.BuildImage(true))
+	AfterAll(e.DestroyContainer())
+})

e2e/docs/docs_test.go

@@ -16,7 +16,6 @@ package docs
 
 import (
 	"fmt"
-	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -65,7 +64,7 @@ var _ = Describe("check examples in documentation", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
-	up_tests := []string{"testdata/minimal", "testdata/getting_started", "testdata/jupyter", "testdata/complex"}
+	up_tests := []string{"testdata/minimal", "testdata/getting_started", "testdata/jupyter"}
 
 	for _, v := range up_tests {
 		It(fmt.Sprintf("can up %s environment", v), func() {
@@ -86,8 +85,3 @@ var _ = Describe("check examples in documentation", func() {
 	}
 
 })
-
-func TestMain(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "envd documentation example test suite")
-}

e2e/docs/suite_test.go

@@ -0,0 +1,34 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docs
+
+import (
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/tensorchord/envd/pkg/version"
+)
+
+func init() {
+	version.SetGitTagForE2ETest(os.Getenv("GIT_LATEST_TAG"))
+}
+
+func TestMain(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "envd cli Suite")
+}

e2e/e2e_helper.go

@@ -28,6 +28,10 @@ import (
 	"github.com/tensorchord/envd/pkg/lang/ir"
 )
 
+func BuildContextDirWithName(name string) string {
+	return filepath.Join("testdata", name)
+}
+
 func ResetEnvdApp() {
 	ir.DefaultGraph = ir.NewGraph()
 }

e2e/language/python_test.go

@@ -24,23 +24,23 @@ var _ = Describe("python", Ordered, func() {
 	It("Should build packages successfully", func() {
 		exampleName := "python/packages"
 		testcase := "e2e"
-		e := e2e.NewExample(buildContextDirWithName(exampleName), testcase)
+		e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 		e.BuildImage(true)()
 		e.RunContainer()()
 		e.DestroyContainer()()
 	})
 	It("Should build requirements successfully", func() {
 		exampleName := "python/requirements"
 		testcase := "e2e"
-		e := e2e.NewExample(buildContextDirWithName(exampleName), testcase)
+		e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 		e.BuildImage(true)()
 		e.RunContainer()()
 		e.DestroyContainer()()
 	})
 	It("Should build hybrid successfully", func() {
 		exampleName := "python/hybrid"
 		testcase := "e2e"
-		e := e2e.NewExample(buildContextDirWithName(exampleName), testcase)
+		e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 		e.BuildImage(true)()
 		e.RunContainer()()
 		e.DestroyContainer()()
@@ -49,7 +49,7 @@ var _ = Describe("python", Ordered, func() {
 	It("Should build conda with channel successfully", func() {
 		exampleName := "python/conda"
 		testcase := "e2e"
-		e := e2e.NewExample(buildContextDirWithName(exampleName), testcase)
+		e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 		e.BuildImage(true)()
 		e.RunContainer()()
 		e.DestroyContainer()()

e2e/language/runtime_test.go

@@ -24,7 +24,7 @@ import (
 var _ = Describe("runtime", Ordered, func() {
 	exampleName := "runtime"
 	testcase := "e2e"
-	e := e2e.NewExample(buildContextDirWithName(exampleName), testcase)
+	e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 	BeforeAll(e.BuildImage(true))
 	BeforeEach(e.RunContainer())
 	It("execute runtime command `numpy`", func() {

e2e/language/suite_test.go

@@ -16,7 +16,6 @@ package language
 
 import (
 	"os"
-	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -25,11 +24,6 @@ import (
 	"github.com/tensorchord/envd/pkg/version"
 )
 
-func buildContextDirWithName(name string) string {
-	basePath := "testdata"
-	return filepath.Join(basePath, name)
-}
-
 func init() {
 	version.SetGitTagForE2ETest(os.Getenv("GIT_LATEST_TAG"))
 }

examples/conda/build.envd

@@ -26,4 +26,4 @@ custom_channels:
     install.python_packages(name = [
         "flask"
     ])
-    install.cuda(version="11.6", cudnn="8")
+    install.cuda(version="11.2", cudnn="8")

examples/dgl/build.envd

@@ -16,7 +16,7 @@ def build_gpu():
     base(os="ubuntu20.04", language="python3")  
 
     # install cuda
-    install.cuda(version="11.6", cudnn="8")
+    install.cuda(version="11.2", cudnn="8")
 
     # Add the packages you are using here
     install.python_packages(["numpy"])

examples/mnist/build.envd

@@ -13,4 +13,4 @@ def build():
 
 def build_gpu():
     build()
-    install.cuda(version="11.6", cudnn="8")
+    install.cuda(version="11.2", cudnn="8")

pkg/app/destroy.go

@@ -15,7 +15,9 @@
 package app
 
 import (
+	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
@@ -72,39 +74,51 @@ func destroy(clicontext *cli.Context) error {
 		ctrName = filepath.Base(buildContext)
 	}
 
-	tag, err := getContainerTag(clicontext, ctrName)
-	if err != nil {
-		return err
-	}
-
 	if ctrName, err := dockerClient.Destroy(clicontext.Context, ctrName); err != nil {
 		return errors.Wrapf(err, "failed to destroy the environment: %s", ctrName)
 	} else if ctrName != "" {
-		logrus.Infof("%s is destroyed", ctrName)
+		logrus.Infof("container(%s) is destroyed", ctrName)
 	}
-	if err := dockerClient.RemoveImage(clicontext.Context, tag); err != nil {
-		return errors.Wrapf(err, "failed to remove the image: %s", tag)
+
+	tags, err := getContainerTag(clicontext, ctrName)
+	if err != nil {
+		return err
+	} else {
+		for _, tag := range tags {
+			if err := dockerClient.RemoveImage(clicontext.Context, tag); err != nil {
+				return errors.Errorf("remove image %s failed: %w", tag, err)
+			}
+			logrus.Infof("image(%s) is destroyed", tag)
+		}
 	}
+
 	if err = sshconfig.RemoveEntry(ctrName); err != nil {
 		logrus.Infof("failed to remove entry %s from your SSH config file: %s", ctrName, err)
 		return errors.Wrap(err, "failed to remove entry from your SSH config file")
 	}
 	return nil
 }
 
-func getContainerTag(clicontext *cli.Context, name string) (string, error) {
+func getContainerTag(clicontext *cli.Context, name string) ([]string, error) {
+	tags := []string{}
 	envdEngine, err := envd.New(clicontext.Context)
 	if err != nil {
-		return "", err
+		return tags, err
 	}
-	envs, err := envdEngine.ListEnvironment(clicontext.Context)
+	// check the images instead of running containers because `envd build` also produce images
+	images, err := envdEngine.ListImage(clicontext.Context)
 	if err != nil {
-		return "", err
+		return tags, err
 	}
-	for _, env := range envs {
-		if env.Name == name {
-			return env.Container.Image, nil
+	for _, img := range images {
+		for _, tag := range img.ImageSummary.RepoTags {
+			if strings.HasPrefix(tag, fmt.Sprintf("%s:", name)) {
+				tags = append(tags, tag)
+			}
 		}
 	}
-	return "", errors.Newf("cannot find the image of %s", name)
+	if len(tags) == 0 {
+		logrus.Infof("cannot find the image of %s", name)
+	}
+	return tags, nil
 }