e2e test for firtlet pod restart

deploy/virtlet-ds.yaml

@@ -107,6 +107,12 @@ spec:
  name: virtlet-config
  key: disable_kvm
  optional: true
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null
  - name: virtlet
  image: mirantis/virtlet
  # In case we inject local virtlet image we want to use it not officially available one
@@ -188,6 +194,12 @@ spec:
  value: /etc/virtlet/images
  - name: KUBERNETES_POD_LOGS
  value: "/kubernetes-log"
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - socat - UNIX:/run/virtlet.sock </dev/null
  - name: vms
  image: mirantis/virtlet
  imagePullPolicy: IfNotPresent

tests/e2e/common.go

@@ -26,6 +26,8 @@ import (
 
  . "github.com/onsi/gomega"
 
+ "k8s.io/api/core/v1"
+
  "github.com/Mirantis/virtlet/tests/e2e/framework"
  . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
 )
@@ -65,6 +67,25 @@ func waitSSH(vm *framework.VMInterface) framework.Executor {
  return ssh
 }
 
+func waitVirtletPod(vm *framework.VMInterface) *framework.PodInterface {
+ var virtletPod *framework.PodInterface
+ Eventually(
+ func() error {
+ var err error
+ virtletPod, err = vm.VirtletPod()
+ if err != nil {
+ return err
+ }
+ for _, c := range virtletPod.Pod.Status.Conditions {
+ if c.Type == v1.PodReady && c.Status == v1.ConditionTrue {
+ return nil
+ }
+ }
+ return fmt.Errorf("Pod not ready yet: %+v", virtletPod.Pod.Status)
+ }, 60*5, 3).Should(Succeed())
+ return virtletPod
+}
+
 func checkCPUCount(vm *framework.VMInterface, ssh framework.Executor, cpus int) {
  proc := do(framework.RunSimple(ssh, "cat", "/proc/cpuinfo")).(string)
  Expect(regexp.MustCompile(`(?m)^processor`).FindAllString(proc, -1)).To(HaveLen(cpus))

tests/e2e/framework/pod_interface.go

@@ -28,7 +28,6 @@ import (
 
  "github.com/davecgh/go-spew/spew"
  "k8s.io/api/core/v1"
- "k8s.io/apimachinery/pkg/api/errors"
  k8serrors "k8s.io/apimachinery/pkg/api/errors"
 
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -69,9 +68,9 @@ func (pi *PodInterface) Create() error {
 // Delete deletes the pod and associated service, which was earlier created by `controller.Run()`
 func (pi *PodInterface) Delete() error {
  if pi.hasService {
- pi.controller.client.Services(pi.controller.Namespace()).Delete(pi.Pod.Name, nil)
+ pi.controller.client.Services(pi.Pod.Namespace).Delete(pi.Pod.Name, nil)
  }
- return pi.controller.client.Pods(pi.controller.Namespace()).Delete(pi.Pod.Name, nil)
+ return pi.controller.client.Pods(pi.Pod.Namespace).Delete(pi.Pod.Name, nil)
 }
 
 // Wait waits for pod to start and checks that it doesn't fail immediately after that

tests/e2e/restart_virtlet_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2017 Mirantis
+Copyright 2018 Mirantis
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,6 +17,9 @@ limitations under the License.
 package e2e
 
 import (
+ "bytes"
+ "context"
+ "fmt"
  "time"
 
  . "github.com/onsi/gomega"
@@ -25,7 +28,7 @@ import (
  . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
 )
 
-var _ = Describe("Virtlet restart", func() {
+var _ = Describe("Virtlet restart [Disruptive]", func() {
  var (
  vm *framework.VMInterface
  vmPod *framework.PodInterface
@@ -37,25 +40,47 @@ var _ = Describe("Virtlet restart", func() {
  var err error
  vmPod, err = vm.Pod()
  Expect(err).NotTo(HaveOccurred())
+
+ // restart virtlet before all tests
+ virtletPod, err := vm.VirtletPod()
+ Expect(err).NotTo(HaveOccurred())
+
+ err = virtletPod.Delete()
+ Expect(err).NotTo(HaveOccurred())
+
+ waitVirtletPod(vm)
  })
 
  AfterAll(func() {
  deleteVM(vm)
  })
 
- It("Should allow to ssh to VM after virtlet pod and vm restart [Conformance]", func() {
- pod, err := vm.VirtletPod()
-  Expect(err).NotTo(HaveOccurred())
+ It("Should allow to ssh to VM after virtlet pod restart", func() {
+ waitSSH(vm)
+ }, 3*60)
 
- virtletPodExecutor, err := pod.Container("virtlet")
- Expect(err).NotTo(HaveOccurred())
+ It("Should keep logs from another session", func() {
+ var stdout bytes.Buffer
+ ctx, closeFunc := context.WithCancel(context.Background())
+ defer closeFunc()
+ localExecutor := framework.LocalExecutor(ctx)
 
- do(framework.ExecSimple(virtletPodExecutor, "pkill", "-9", "virtlet"))
- Expect(pod.Wait()).NotTo(HaveOccurred())
+ By(fmt.Sprintf("Running command: kubectl logs -n %s %s", controller.Namespace(), vm.Name))
+ err := localExecutor.Run(nil, &stdout, &stdout, "kubectl", "-n", controller.Namespace(), "logs", vm.Name)
+ fmt.Sprintf(stdout.String())
+ Expect(err).NotTo(HaveOccurred())
+ Expect(stdout.String()).Should(ContainSubstring("login as 'cirros' user."))
 
- _, err = vm.VirshCommand("reboot", "<domain>")
+ By(fmt.Sprintf("Running command: kubectl attach -n %s -i %s", controller.Namespace(), vm.Name))
+ stdin := bytes.NewBufferString("\nTESTTEXT\n\n")
+ stdout.Reset()
+ err = localExecutor.Run(stdin, &stdout, &stdout, "kubectl", "-n", controller.Namespace(), "attach", "-i", vm.Name)
  Expect(err).NotTo(HaveOccurred())
 
- waitSSH(vm)
+ By(fmt.Sprintf("Running again command: kubectl logs -n %s %s", controller.Namespace(), vm.Name))
+ stdout.Reset()
+ err = localExecutor.Run(nil, &stdout, &stdout, "kubectl", "-n", controller.Namespace(), "logs", vm.Name)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(stdout.String()).Should(ContainSubstring("TESTTEXT"))
  }, 3*60)
 })