feat: Add 'prefetch' extension for the generating suites

.golangci.yml

@@ -162,3 +162,13 @@ linters:
     # - unused
     - varcheck
     - whitespace
+issues:
+  exclude-use-default: false
+  max-issues-per-linter: 0
+  max-same-issues: 0
+  exclude-rules:
+    # We really *do* want to pass a pointer to an interface in these cases.  See comments in file for New{Server,Client}
+    # function for why
+    - path: "extensions/prefetch/sriov/sriov.go"
+      linters:
+        - gochecknoinits

extensions/base/suite.go

@@ -14,38 +14,54 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package base exports base suite type that will be injected into each generated suite.
 package base
 
 import (
+	"github.com/networkservicemesh/gotestmd/pkg/suites/shell"
 	"github.com/networkservicemesh/integration-tests/extensions/checkout"
 	"github.com/networkservicemesh/integration-tests/extensions/logs"
+	"github.com/networkservicemesh/integration-tests/extensions/prefetch"
 )
 
 // Suite is a base suite for generating tests. Contains extensions that can be used for assertion and automation goals.
 type Suite struct {
-	checkout.Suite
-	storeTestLogs, storeSuiteLogs func()
+	shell.Suite
 	// Add other extensions here
+	checkout                      checkout.Suite
+	prefetch                      prefetch.Suite
+	storeTestLogs, storeSuiteLogs func()
 }
 
-func (s *Suite) SetupSuite() {
-	s.Repository = "networkservicemesh/deployments-k8s"
-	s.Version = "dcaced16"
-	s.Dir = "../" // Note: this should be synced with input parameters in gen.go file
-
-	s.Suite.SetupSuite()
-
-	s.storeSuiteLogs = logs.Capture(s.T().Name())
-}
-
+// AfterTest stores logs after each test in the suite.
 func (s *Suite) AfterTest(_, _ string) {
 	s.storeTestLogs()
 }
 
+// BeforeTest starts capture logs for each test in the suite.
 func (s *Suite) BeforeTest(_, _ string) {
 	s.storeTestLogs = logs.Capture(s.T().Name())
 }
 
+// TearDownSuite stores logs from containers that spawned during SuiteSetup.
 func (s *Suite) TearDownSuite() {
 	s.storeSuiteLogs()
 }
+
+// SetupSuite runs all extensions
+func (s *Suite) SetupSuite() {
+	s.checkout.Repository = "networkservicemesh/deployments-k8s"
+	s.checkout.Version = "0530e54c"
+	s.checkout.Dir = "../" // Note: this should be synced with input parameters in gen.go file
+
+	s.checkout.SetT(s.T())
+	s.checkout.SetupSuite()
+
+	// prefetch
+	s.prefetch.Dir = "../deployments-k8s" // Note: this should be synced with input parameters in gen.go file
+
+	s.prefetch.SetT(s.T())
+	s.prefetch.SetupSuite()
+
+	s.storeSuiteLogs = logs.Capture(s.T().Name())
+}

extensions/checkout/suite.go

@@ -14,6 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package checkout contains a suite that checkouts missed repository.
 package checkout
 
 import (
@@ -35,11 +36,13 @@ type Suite struct {
 
 const urlFormat = "https://github.com/%v.git"
 
+// SetupSuite clones repository if it is not presented on the local machine.
 func (s *Suite) SetupSuite() {
 	r := s.Runner(s.Dir)
 	u := fmt.Sprintf(urlFormat, s.Repository)
 	_, dir := path.Split(s.Repository)
 	repoDir := filepath.Join(r.Dir(), dir)
+	// #nosec
 	if _, err := os.Open(repoDir); err != nil {
 		r.Run("git clone " + u)
 		r.Run("cd " + repoDir)

extensions/logs/logs.go

@@ -13,6 +13,8 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+// Package logs exports helper functions for storing logs from containers.
 package logs
 
 import (
@@ -48,6 +50,7 @@ var (
 	matchRegex *regexp.Regexp
 )
 
+// Config is env config to setup log collecting.
 type Config struct {
 	KubeConfig        string        `default:"" desc:".kube config file path" envconfig:"KUBECONFIG"`
 	ArtifactsDir      string        `default:"logs" desc:"Directory for storing container logs" envconfig:"ARTIFACTS_DIR"`

extensions/prefetch/ignore.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2021 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prefetch
+
+import "regexp"
+
+var (
+	// ExcludeRegex is using for filtering applications that should not be used in the prefetching.
+	ExcludeRegex = regexp.MustCompile("(.*-sriov)|(.*-vfio)")
+)

extensions/prefetch/k8s.go

@@ -0,0 +1,102 @@
+// Copyright (c) 2021 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prefetch
+
+const createNamespace = `
+cat >prefetch-namespace.yaml <<EOF
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name:prefetch
+EOF
+`
+
+const createConfigMap = `
+cat >prefetch-configmap.yaml <<EOF
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name:prefetch
+data:
+ prefetch.sh: |
+    #!/bin/sh
+
+    for image in {{.TestImages}}; do
+      if ! ctr -n=k8s.io image ls -q | grep "\${image}"; then
+        ctr -n=k8s.io image pull "\${image}"
+      fi
+    done
+EOF
+`
+
+const createDaemonSet = `
+cat >prefetch.yaml <<EOF
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name:prefetch
+  labels:
+    app:prefetch
+spec:
+  selector:
+    matchLabels:
+      app:prefetch
+  template:
+    metadata:
+      labels:
+        app:prefetch
+    spec:
+      initContainers:
+        - name:prefetch
+          image: docker:latest
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "/root/scripts/ctr-pull.sh"]
+          volumeMounts:
+            - name: containerd
+              mountPath: /run/containerd/containerd.sock
+            - name: scripts
+              mountPath: /root/scripts
+      containers:
+        - name: pause
+          image: google/pause:latest
+      volumes:
+        - name: containerd
+          hostPath:
+            path: /run/containerd/containerd.sock
+        - name: scripts
+          configMap:
+            name:prefetch
+EOF
+`
+
+const createKustomization = `
+cat > kustomization.yaml <<EOF
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace:prefetch
+
+resources:
+-prefetch-namespace.yaml
+-prefetch-configmap.yaml
+-prefetch.yaml
+EOF
+`

extensions/prefetch/sriov/sriov.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2021 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package sriov can be used to enable prefetching of SRIOV related applications
+package sriov
+
+import (
+	"regexp"
+
+	"github.com/networkservicemesh/integration-tests/extensions/prefetch"
+)
+
+func init() {
+	prefetch.ExcludeRegex = regexp.MustCompile(".*")
+}