Merge pull request networkservicemesh#891 from Bolodya1997/integratio…

…n-tests#864/prefetch [integration tests#864] Rework prefetch
Bolodya1997 · Oct 12, 2021 · 51f1fd5 · 51f1fd5
2 parents d7b4ea9 + ce5b406
commit 51f1fd5
Show file tree

Hide file tree

Showing 7 changed files with 147 additions and 159 deletions.
diff --git a/extensions/base/suite.go b/extensions/base/suite.go
@@ -18,6 +18,8 @@
 package base
 
 import (
+	"fmt"
+
 	"github.com/networkservicemesh/gotestmd/pkg/suites/shell"
 	"github.com/networkservicemesh/integration-tests/extensions/checkout"
 	"github.com/networkservicemesh/integration-tests/extensions/logs"
@@ -48,17 +50,21 @@ func (s *Suite) TearDownSuite() {
 	s.storeSuiteLogs()
 }
 
+const (
+	sha = "61e3eb260f1fc0f66e4d7bdab513fd7cecd8c0d7"
+)
+
 // SetupSuite runs all extensions
 func (s *Suite) SetupSuite() {
 	s.checkout.Repository = "networkservicemesh/deployments-k8s"
-	s.checkout.Version = "bf12efe5"
+	s.checkout.Version = sha[:8]
 	s.checkout.Dir = "../" // Note: this should be synced with input parameters in gen.go file
 
 	s.checkout.SetT(s.T())
 	s.checkout.SetupSuite()
 
 	// prefetch
-	s.prefetch.Dir = "../deployments-k8s" // Note: this should be synced with input parameters in gen.go file
+	s.prefetch.ImagesURL = fmt.Sprintf("https://raw.githubusercontent.com/networkservicemesh/deployments-k8s/%s/images.yaml", sha)
 
 	s.prefetch.SetT(s.T())
 	s.prefetch.SetupSuite()

diff --git a/extensions/prefetch/ignore.go b/extensions/prefetch/ignore.go
@@ -16,9 +16,7 @@
 
 package prefetch
 
-import "regexp"
-
 var (
-	// IsExcluded is using for filtering applications that should not be used in the prefetching.
-	IsExcluded = regexp.MustCompile("(.*-sriov)|(.*-vfio)").MatchString
+	// Tags is used for filtering images that should not be used in the prefetching.
+	Tags = make(map[string]struct{})
 )
diff --git a/extensions/prefetch/k8s.go b/extensions/prefetch/k8s.go
@@ -16,87 +16,79 @@
 
 package prefetch
 
-const createNamespace = `
-cat >prefetch-namespace.yaml <<EOF
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: prefetch
-EOF
-`
+import (
+	"bytes"
+	"text/template"
+)
 
-const createConfigMap = `
-cat >prefetch-configmap.yaml <<EOF
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: prefetch
-data:
- prefetch.sh: |
-    #!/bin/sh
-
-    for image in {{.TestImages}}; do
-      if ! ctr -n=k8s.io image ls -q | grep "\${image}"; then
-        ctr -n=k8s.io image pull "\${image}"
-      fi
-    done
-EOF
-`
-
-const createDaemonSet = `
-cat >prefetch.yaml <<EOF
+func createDaemonSet(number int, containers string) string {
+	const text = `
+cat > prefetch-{{.Number}}.yaml <<EOF
 ---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: prefetch
+  name: prefetch-{{.Number}}
   labels:
-    app: prefetch
+    app: prefetch-{{.Number}}
 spec:
   selector:
     matchLabels:
-      app: prefetch
+      app: prefetch-{{.Number}}
   template:
     metadata:
       labels:
-        app: prefetch
+        app: prefetch-{{.Number}}
     spec:
       initContainers:
-        - name: prefetch
-          image: docker:latest
+        - name: return
+          image: rrandom312/return
           imagePullPolicy: IfNotPresent
-          command: ["/bin/sh", "/root/scripts/prefetch.sh"]
+          command: ["cp", "/bin/return", "/out/return"]
           volumeMounts:
-            - name: containerd
-              mountPath: /run/containerd/containerd.sock
-            - name: scripts
-              mountPath: /root/scripts
+            - name: bin
+              mountPath: /out
+{{.Containers}}
       containers:
         - name: pause
           image: google/pause:latest
       volumes:
-        - name: containerd
-          hostPath:
-            path: /run/containerd/containerd.sock
-        - name: scripts
-          configMap:
-            name: prefetch
+        - name: bin
+          emptyDir: { }
 EOF
 `
+	return substitute(text, &struct {
+		Number     int
+		Containers string
+	}{
+		Number:     number,
+		Containers: containers,
+	})
+}
 
-const createKustomization = `
-cat > kustomization.yaml <<EOF
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
+func container(name, image string) string {
+	const text = `
+        - name: {{.Name}}
+          image: {{.Image}}
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/return"]
+          volumeMounts:
+            - name: bin
+              mountPath: /bin
+`
+	return substitute(text, &struct {
+		Name, Image string
+	}{
+		Name:  name,
+		Image: image,
+	})
+}
 
-namespace: prefetch
+func substitute(text string, data interface{}) string {
+	t, _ := template.New("").Parse(text)
 
-resources:
-- prefetch-namespace.yaml
-- prefetch-configmap.yaml
-- prefetch.yaml
-EOF
-`
+	buf := new(bytes.Buffer)
+	_ = t.Execute(buf, data)
+
+	return buf.String()
+}
diff --git a/extensions/prefetch/sriov/sriov.go b/extensions/prefetch/sriov/sriov.go
@@ -22,7 +22,5 @@ import (
 )
 
 func init() {
-	prefetch.IsExcluded = func(_ string) bool {
-		return false
-	}
+	prefetch.Tags["sriov"] = struct{}{}
 }
diff --git a/extensions/prefetch/suite.go b/extensions/prefetch/suite.go
@@ -19,131 +19,117 @@ package prefetch
 
 import (
 	"bufio"
+	"fmt"
+	"net/http"
 	"os"
-	"path/filepath"
-	"regexp"
 	"strings"
 	"sync"
 
 	"github.com/google/uuid"
+	"github.com/kelseyhightower/envconfig"
 	"github.com/stretchr/testify/require"
 
 	"github.com/networkservicemesh/gotestmd/pkg/suites/shell"
 )
 
-const (
-	defaultDomain = "docker.io"
-	officialLib   = "library"
-	defaultTag    = ":latest"
-)
+// Config is env config to setup images prefetching.
+type Config struct {
+	ImagesPerDaemonset int    `default:"10" desc:"Number of images created per DaemonSet" split_words:"true"`
+	Timeout            string `default:"10m" desc:"Kubectl rollout status timeout for the DaemonSet" split_words:"true"`
+}
 
 // Suite creates `prefetch` daemonset which pulls all test images for all cluster nodes.
 type Suite struct {
 	shell.Suite
-	Dir string
+	ImagesURL string
 }
 
 var once sync.Once
 
 // SetupSuite prefetches docker images for each k8s node.
 func (s *Suite) SetupSuite() {
-	once.Do(func() {
-		testImages, err := s.findTestImages()
-		require.NoError(s.T(), err)
-
-		tmpDir := uuid.NewString()
-		require.NoError(s.T(), os.MkdirAll(tmpDir, 0750))
-
-		r := s.Runner(tmpDir)
+	once.Do(s.initialize)
+}
 
-		r.Run(createNamespace)
-		r.Run(strings.ReplaceAll(createConfigMap, "{{.TestImages}}", strings.Join(testImages, " ")))
-		r.Run(createDaemonSet)
-		r.Run(createKustomization)
+func (s *Suite) initialize() {
+	var config Config
+	require.NoError(s.T(), envconfig.Usage("prefetch", &config))
+	require.NoError(s.T(), envconfig.Process("prefetch", &config))
 
-		r.Run("kubectl apply -k .")
-		r.Run("kubectl -n prefetch wait --timeout=10m --for=condition=ready pod -l app=prefetch")
+	images := s.findImages()
 
-		r.Run("kubectl delete ns prefetch")
-		_ = os.RemoveAll(tmpDir)
-	})
-}
+	tmpDir := uuid.NewString()
+	require.NoError(s.T(), os.MkdirAll(tmpDir, 0750))
+	s.T().Cleanup(func() { _ = os.RemoveAll(tmpDir) })
 
-func (s *Suite) findTestImages() ([]string, error) {
-	imagePattern := regexp.MustCompile(".*image: (?P<image>.*)")
-	imageSubexpIndex := imagePattern.SubexpIndex("image")
+	r := s.Runner(tmpDir)
 
-	var testImages []string
-	walkFunc := func(path string, info os.FileInfo, err error) error {
-		if ok, skipErr := s.shouldSkipWithError(info, err); ok {
-			return skipErr
-		}
-		// #nosec
-		file, err := os.Open(path)
-		if err != nil {
-			return err
+	var daemonSets []string
+	for d := 0; d*config.ImagesPerDaemonset < len(images); d++ {
+		var containers string
+		for c := 0; c < config.ImagesPerDaemonset && d*config.ImagesPerDaemonset+c < len(images); c++ {
+			containers += container(uuid.NewString(), images[d*config.ImagesPerDaemonset+c])
 		}
 
-		scanner := bufio.NewScanner(file)
-		for scanner.Scan() {
-			if imagePattern.MatchString(scanner.Text()) {
-				image := imagePattern.FindAllStringSubmatch(scanner.Text(), -1)[0][imageSubexpIndex]
-				testImages = append(testImages, s.fullImageName(image))
-			}
-		}
+		r.Run(createDaemonSet(d, containers))
 
-		return nil
+		daemonSets = append(daemonSets, fmt.Sprintf("prefetch-%d", d))
 	}
 
-	if err := filepath.Walk(filepath.Join(s.Dir, "apps"), walkFunc); err != nil {
-		return nil, err
-	}
-	if err := filepath.Walk(filepath.Join(s.Dir, "examples", "spire"), walkFunc); err != nil {
-		return nil, err
-	}
+	r.Run("kubectl create ns prefetch")
+	s.T().Cleanup(func() { r.Run("kubectl delete ns prefetch") })
 
-	return testImages, nil
-}
+	var wg sync.WaitGroup
+	for _, daemonSet := range daemonSets {
+		wg.Add(1)
+		go func(daemonSet string) {
+			defer wg.Done()
 
-func (s *Suite) shouldSkipWithError(info os.FileInfo, err error) (bool, error) {
-	if err != nil {
-		return true, err
-	}
-
-	if info.IsDir() {
-		if IsExcluded(info.Name()) {
-			return true, filepath.SkipDir
-		}
-		return true, nil
+			dr := s.Runner(tmpDir)
+			dr.Run(fmt.Sprintf("kubectl -n prefetch apply -f %s.yaml", daemonSet))
+			dr.Run(fmt.Sprintf("kubectl -n prefetch rollout status daemonset/%s --timeout=%s", daemonSet, config.Timeout))
+			dr.Run(fmt.Sprintf("kubectl -n prefetch delete -f %s.yaml", daemonSet))
+		}(daemonSet)
 	}
+	wg.Wait()
+}
 
-	if !strings.HasSuffix(info.Name(), ".yaml") {
-		return true, nil
-	}
+func (s *Suite) findImages() (images []string) {
+	in, err := http.Get(s.ImagesURL)
+	require.NoError(s.T(), err)
+	defer func() { _ = in.Body.Close() }()
 
-	return false, nil
-}
+	for scanner := bufio.NewScanner(in.Body); scanner.Scan(); {
+		line := strings.TrimSpace(scanner.Text())
+		if !strings.HasPrefix(line, "- ") {
+			continue
+		}
+		line = strings.TrimPrefix(line, "- ")
+
+		var image string
+		var tags []string
+		switch split := strings.Split(line, "#"); {
+		case len(split) > 2:
+			require.Fail(s.T(), "line is invalid: %s", line)
+		case len(split) == 2:
+			tags = strings.Split(strings.TrimSpace(split[1]), ",")
+			fallthrough
+		default:
+			image = strings.TrimSpace(split[0])
+		}
 
-func (s *Suite) fullImageName(image string) string {
-	var domain, remainder string
-	i := strings.IndexRune(image, '/')
-	if i == -1 || (!strings.ContainsAny(image[:i], ".:")) {
-		domain, remainder = defaultDomain, image
-	} else {
-		domain, remainder = image[:i], image[i+1:]
-	}
-	if domain == defaultDomain && !strings.ContainsRune(remainder, '/') {
-		remainder = officialLib + "/" + remainder
-	}
+		if len(tags) == 0 {
+			images = append(images, image)
+			continue
+		}
 
-	switch len(strings.Split(remainder, ":")) {
-	case 2:
-		// nothing to do
-	case 1:
-		remainder += defaultTag
-	default:
-		return ""
+		for _, tag := range tags {
+			if _, ok := Tags[tag]; ok {
+				images = append(images, image)
+				break
+			}
+		}
 	}
 
-	return domain + "/" + remainder
+	return images
 }