Add containers log saver extension

extensions/base/suite.go

@@ -18,17 +18,24 @@ package base
 
 import (
 	"github.com/networkservicemesh/integration-tests/extensions/checkout"
+	"github.com/networkservicemesh/integration-tests/extensions/logs"
+	"github.com/networkservicemesh/integration-tests/extensions/multisuite"
 )
 
 // Suite is a base suite for generating tests. Contains extensions that can be used for assertion and automation goals.
 type Suite struct {
-	checkout.Suite
-	// Add other extensions here
+	multisuite.Suite
 }
 
 func (s *Suite) SetupSuite() {
-	s.Repository = "networkservicemesh/deployments-k8s"
-	s.Version = "e2954268"
-	s.Dir = "../" // Note: this should be synced with input parameters in gen.go file
+	// Add other extensions here
+	s.Suite = multisuite.New(s.T(),
+		new(logs.Suite),
+		&checkout.Suite{
+			Repository: "networkservicemesh/deployments-k8s",
+			Version:    "e2954268",
+			Dir:        "../", // Note: this should be synced with input parameters in gen.go file
+		})
+
 	s.Suite.SetupSuite()
 }

extensions/logs/suite.go

@@ -0,0 +1,234 @@
+// Copyright (c) 2021 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logs
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/kelseyhightower/envconfig"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+const (
+	allNamespaces = ""
+	configPrefix  = "LOG"
+)
+
+var instanceExists = false
+var initConfig sync.Once
+var config Config
+
+type Config struct {
+	KubeConfig   string `default:"~/.kube/config" desc:"Kubernetes configuration file" envconfig:"KUBECONFIG"`
+	ArtifactsDir string `default:"logs" desc:"Directory for storing container logs" envconfig:"ARTIFACTS_DIR"`
+
+	ExcludeK8sNs   []string      `default:"kube-system,local-path-storage" desc:"Comma-separated list of excluded kubernetes namespaces" split_words:"true"`
+	ContextTimeout time.Duration `default:"15s" desc:"Context timeout for kubernetes queries" split_words:"true"`
+	WorkerCount    int           `default:"4" desc:"Number of log collector workers" split_words:"true"`
+}
+
+type Suite struct {
+	suite.Suite
+
+	kubeClient kubernetes.Interface
+
+	firstInstance bool
+	testStartTime time.Time
+	nsmContainers map[types.UID]bool
+	logQueue      chan logItem
+	waitGroup     sync.WaitGroup
+}
+
+type logItem struct {
+	namespace  string
+	pod        string
+	logDir     string
+	logOptions *corev1.PodLogOptions
+}
+
+func (s *Suite) SetupSuite() {
+	initConfig.Do(func() {
+		if err := envconfig.Usage(configPrefix, &config); err != nil {
+			panic(err)
+		}
+
+		if err := envconfig.Process(configPrefix, &config); err != nil {
+			panic(err)
+		}
+	})
+
+	if instanceExists {
+		return
+	}
+
+	instanceExists = true
+	s.firstInstance = true
+	var err error
+
+	s.kubeClient, err = newKubeClient()
+	require.NoError(s.T(), err)
+
+	s.logQueue = make(chan logItem)
+	for i := 0; i < config.WorkerCount; i++ {
+		s.waitGroup.Add(1)
+		go func() {
+			defer s.waitGroup.Done()
+			for src := range s.logQueue {
+				s.saveLog(src)
+			}
+		}()
+	}
+}
+
+func (s *Suite) TearDownSuite() {
+	if !s.firstInstance {
+		return
+	}
+
+	instanceExists = false
+	close(s.logQueue)
+	s.waitGroup.Wait()
+}
+
+func (s *Suite) SetupTest() {
+	if !s.firstInstance {
+		return
+	}
+
+	s.testStartTime = time.Now()
+	s.nsmContainers = make(map[types.UID]bool)
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.ContextTimeout)
+	defer cancel()
+
+	pods, err := s.kubeClient.CoreV1().Pods(allNamespaces).List(ctx, metav1.ListOptions{})
+	require.NoError(s.T(), err)
+
+	for podIdx := range pods.Items {
+		pod := &pods.Items[podIdx]
+
+		if !isExcludedNamespace(pod.Namespace) {
+			s.nsmContainers[pod.UID] = true
+		}
+	}
+}
+
+func (s *Suite) AfterTest(suiteName, testName string) {
+	if !s.firstInstance {
+		return
+	}
+
+	logDir := filepath.Join(config.ArtifactsDir, suiteName, testName)
+	require.NoError(s.T(), os.MkdirAll(logDir, os.ModePerm))
+
+	logOptions := corev1.PodLogOptions{
+		Timestamps: true,
+		SinceTime:  &metav1.Time{Time: s.testStartTime},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.ContextTimeout)
+	defer cancel()
+
+	pods, err := s.kubeClient.CoreV1().Pods(allNamespaces).List(ctx, metav1.ListOptions{})
+	require.NoError(s.T(), err)
+
+	var waitGroup sync.WaitGroup
+	for podIdx := range pods.Items {
+		pod := &pods.Items[podIdx]
+
+		if isExcludedNamespace(pod.Namespace) {
+			continue
+		}
+
+		nextLogItem := logItem{
+			namespace:  pod.Namespace,
+			pod:        pod.Name,
+			logDir:     logDir,
+			logOptions: &logOptions,
+		}
+
+		if _, ok := s.nsmContainers[pod.UID]; ok {
+			s.logQueue <- nextLogItem
+			continue
+		}
+
+		waitGroup.Add(1)
+		go func() {
+			defer waitGroup.Done()
+			s.saveLog(nextLogItem)
+		}()
+	}
+
+	waitGroup.Wait()
+}
+
+func (s *Suite) saveLog(src logItem) {
+	ctx, cancel := context.WithTimeout(context.Background(), config.ContextTimeout)
+	defer cancel()
+
+	data, err := s.kubeClient.CoreV1().
+		Pods(src.namespace).
+		GetLogs(src.pod, src.logOptions).
+		DoRaw(ctx)
+
+	require.NoError(s.T(), err)
+
+	if len(data) > 0 {
+		logFile := filepath.Join(src.logDir, src.pod+".log")
+		require.NoError(s.T(), ioutil.WriteFile(logFile, data, os.ModePerm))
+	}
+}
+
+// newKubeClient creates new k8s client
+func newKubeClient() (kubernetes.Interface, error) {
+	if strings.HasPrefix(config.KubeConfig, "~") {
+		config.KubeConfig = strings.Replace(config.KubeConfig, "~", os.Getenv("HOME"), 1)
+	}
+
+	kubeconfig, err := clientcmd.BuildConfigFromFlags("", config.KubeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	kubeconfig.Burst = config.WorkerCount * 100
+	kubeconfig.QPS = float32(config.WorkerCount) * 100
+
+	return kubernetes.NewForConfig(kubeconfig)
+}
+
+func isExcludedNamespace(ns string) bool {
+	for _, excluded := range config.ExcludeK8sNs {
+		if ns == excluded {
+			return true
+		}
+	}
+
+	return false
+}

extensions/multisuite/suite.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2021 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multisuite
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/networkservicemesh/gotestmd/pkg/suites/shell"
+)
+
+type Suite struct {
+	shell.Suite
+
+	suits []suite.TestingSuite
+}
+
+func New(t *testing.T, suits ...suite.TestingSuite) Suite {
+	s := Suite{suits: suits}
+	s.SetT(t)
+	return s
+}
+
+func (s *Suite) SetT(t *testing.T) {
+	s.Suite.SetT(t)
+	for _, p := range s.suits {
+		if v, ok := p.(suite.TestingSuite); ok {
+			v.SetT(t)
+		}
+	}
+}
+
+func (s *Suite) SetupSuite() {
+	for _, p := range s.suits {
+		if v, ok := p.(suite.SetupAllSuite); ok {
+			v.SetupSuite()
+		}
+	}
+}
+
+func (s *Suite) SetupTest() {
+	for _, p := range s.suits {
+		if v, ok := p.(suite.SetupTestSuite); ok {
+			v.SetupTest()
+		}
+	}
+}
+
+func (s *Suite) TearDownSuite() {
+	for _, p := range s.suits {
+		if v, ok := p.(suite.TearDownAllSuite); ok {
+			v.TearDownSuite()
+		}
+	}
+}
+
+func (s *Suite) TearDownTest() {
+	for _, p := range s.suits {
+		if v, ok := p.(suite.TearDownTestSuite); ok {
+			v.TearDownTest()
+		}
+	}
+}
+
+func (s *Suite) BeforeTest(suiteName, testName string) {
+	for _, p := range s.suits {
+		if v, ok := p.(suite.BeforeTest); ok {
+			v.BeforeTest(suiteName, testName)
+		}
+	}
+}
+
+func (s *Suite) AfterTest(suiteName, testName string) {
+	for _, p := range s.suits {
+		if v, ok := p.(suite.AfterTest); ok {
+			v.AfterTest(suiteName, testName)
+		}
+	}
+}

go.mod

@@ -3,8 +3,12 @@ module github.com/networkservicemesh/integration-tests
 go 1.15
 
 require (
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/networkservicemesh/gotestmd v0.0.0-20210318180458-eadec9c69fee
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/stretchr/testify v1.6.1
 	golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e // indirect
+	k8s.io/api v0.19.4
+	k8s.io/apimachinery v0.19.4
+	k8s.io/client-go v0.19.4
 )