Add 9pfs e2e test

It doesn't support CirrOS at this moment because of lack of 9p
filesystem support there.

tests/e2e/9pfs_test.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2019 Mirantis
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "path/filepath"
+ "time"
+
+ . "github.com/onsi/gomega"
+
+ "github.com/Mirantis/virtlet/tests/e2e/framework"
+ . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
+ "k8s.io/api/core/v1"
+)
+
+const (
+ bbName = "busybox-9pfs"
+)
+
+var _ = Describe("9pfs volumes [Heavy]", func() {
+ var busyboxPod *framework.PodInterface
+ var vm *framework.VMInterface
+ var ssh framework.Executor
+
+ AfterAll(func() {
+ if ssh != nil {
+ ssh.Close()
+ }
+ if vm != nil {
+ deleteVM(vm)
+ }
+ Expect(busyboxPod.Delete()).To(Succeed())
+ })
+
+ It("Should work with hostPath volumes", func() {
+ if UsingCirros() {
+ Skip("9pfs can't be tested using CirrOS at the moment")
+ }
+
+ By("Picking a Virtlet node")
+ nodeName, err := controller.VirtletNodeName()
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Starting a busybox pod")
+ busyboxPod, err = controller.RunPod(
+ bbName, framework.BusyboxImage,
+ framework.RunPodOptions{
+ Command: []string{"/bin/sleep", "1200"},
+ NodeName: nodeName,
+ HostPathMounts: []framework.HostPathMount{
+ {
+ HostPath: "/tmp",
+ ContainerPath: "/tmp",
+ },
+ },
+ })
+ Expect(err).NotTo(HaveOccurred())
+ Expect(busyboxPod).NotTo(BeNil())
+ bbExec, err := busyboxPod.Container(bbName)
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Creating a temp directory")
+ dir, err := framework.RunSimple(bbExec, "/bin/sh", "-c", "d=`mktemp -d`; echo foo >$d/bar; echo $d")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(dir).NotTo(BeEmpty())
+
+ By("Creating a VM that has temp directory mounted via 9pfs")
+ vm = controller.VM("vm-9pfs")
+ podCustomization := func(pod *framework.PodInterface) {
+ pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{
+ Name: "9pfs-vol",
+ VolumeSource: v1.VolumeSource{
+ HostPath: &v1.HostPathVolumeSource{
+ Path: dir,
+ },
+ },
+ })
+ pod.Pod.Spec.Containers[0].VolumeMounts = append(
+ pod.Pod.Spec.Containers[0].VolumeMounts,
+ v1.VolumeMount{
+ Name: "9pfs-vol",
+ MountPath: "/hostmount",
+ })
+ }
+ Expect(vm.CreateAndWait(VMOptions{
+ NodeName: nodeName,
+ }.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
+ _, err = vm.Pod()
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Wait for the volume to be mounted inside the VM")
+ ssh = waitSSH(vm)
+ Eventually(func() error {
+ _, err := framework.RunSimple(ssh, "sudo test -e /hostmount/bar")
+ return err
+ }, 60*5, 3).Should(Succeed())
+
+ By("Make a copy of a file on the volume inside the VM")
+ _, err = framework.RunSimple(ssh, "sudo cp /hostmount/bar /hostmount/bar1")
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Verifying the new file contents inside the busybox pod")
+ content, err := framework.RunSimple(bbExec, "cat", filepath.Join(dir, "bar1"))
+ Expect(err).NotTo(HaveOccurred())
+ Expect(content).To(Equal("foo"))
+ })
+})

tests/e2e/basic_test.go

@@ -131,7 +131,11 @@ var _ = Describe("Virtlet [Basic cirros tests]", func() {
  podName := "nginx-pf"
 
  By(fmt.Sprintf("Starting nginx pod"))
- nginxPod, err := controller.RunPod(podName, framework.NginxImage, nil, time.Minute*4, 80)
+ nginxPod, err := controller.RunPod(
+ podName, framework.NginxImage,
+ framework.RunPodOptions{
+ ExposePorts: []int32{80},
+ })
  Expect(err).NotTo(HaveOccurred())
  Expect(nginxPod).NotTo(BeNil())
 
@@ -202,7 +206,11 @@ func itShouldHaveNetworkConnectivity(podIface func() *framework.PodInterface, ss
  var nginxPod *framework.PodInterface
 
  BeforeAll(func() {
- p, err := controller.RunPod("nginx", framework.NginxImage, nil, time.Minute*4, 80)
+ p, err := controller.RunPod(
+ "nginx", framework.NginxImage,
+ framework.RunPodOptions{
+ ExposePorts: []int32{80},
+ })
  Expect(err).NotTo(HaveOccurred())
  Expect(p).NotTo(BeNil())
  nginxPod = p

tests/e2e/common.go

@@ -51,6 +51,12 @@ var (
  controller *framework.Controller
 )
 
+// UsingCirros() returns true if cirros image is being used for tests
+// (which has some limitations)
+func UsingCirros() bool {
+ return strings.Contains(*vmImageLocation, "cirros")
+}
+
 // scheduleWaitSSH schedules SSH interface initialization before the test context starts
 func scheduleWaitSSH(vm **framework.VMInterface, ssh *framework.Executor) {
  BeforeAll(func() {

tests/e2e/framework/common.go

@@ -25,7 +25,8 @@ import (
 )
 
 const (
- NginxImage = "nginx:1.14"
+ NginxImage = "docker.io/nginx:1.14.2"
+ BusyboxImage = "docker.io/busybox:1.30.0"
 )
 
 // ErrTimeout is the timeout error returned from functions wrapped by WithTimeout

tests/e2e/framework/controller.go

@@ -51,14 +51,38 @@ import (
 
  virtlet_v1 "github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
  virtletclientv1 "github.com/Mirantis/virtlet/pkg/client/clientset/versioned/typed/virtlet.k8s/v1"
+ "github.com/Mirantis/virtlet/pkg/utils"
 )
 
 const (
- retries = 5
+ retries = 5
+ defaultRunPodTimeout = 4 * time.Minute
 )
 
 var ClusterURL = flag.String("cluster-url", "http://127.0.0.1:8080", "apiserver URL")
 
+// HostPathMount specifies a host path to mount into a pod sandbox.
+type HostPathMount struct {
+ // The path on the host.
+ HostPath string
+ // The path inside the container.
+ ContainerPath string
+}
+
+// RunPodOptions specifies the options for RunPod
+type RunPodOptions struct {
+ // The command to run (optional).
+ Command []string
+ // Timeout. Defaults to 4 minutes.
+ Timeout time.Duration
+ // The list of ports to expose.
+ ExposePorts []int32
+ // The list of host paths to mount.
+ HostPathMounts []HostPathMount
+ // Node name to run this pod on.
+ NodeName string
+}
+
 // Controller is the entry point for various operations on k8s+virtlet entities
 type Controller struct {
  fixedNs bool
@@ -363,31 +387,20 @@ func (c *Controller) Namespace() string {
 }
 
 // RunPod is a helper method to create a pod in a simple configuration (similar to `kubectl run`)
-func (c *Controller) RunPod(name, image string, command []string, timeout time.Duration, exposePorts ...int32) (*PodInterface, error) {
- pod := &v1.Pod{
- ObjectMeta: metav1.ObjectMeta{
- Name: name,
- Labels: map[string]string{"id": name},
- },
- Spec: v1.PodSpec{
- Containers: []v1.Container{
- {
- Name: name,
- Image: image,
- ImagePullPolicy: v1.PullIfNotPresent,
- Command: command,
- },
- },
- },
+func (c *Controller) RunPod(name, image string, opts RunPodOptions) (*PodInterface, error) {
+ if opts.Timeout == 0 {
+ opts.Timeout = defaultRunPodTimeout
  }
+ pod := generatePodSpec(name, image, opts)
+ fmt.Printf("POD:\n%s\n", utils.ToJSON(pod))
  podInterface := newPodInterface(c, pod)
  if err := podInterface.Create(); err != nil {
  return nil, err
  }
- if err := podInterface.Wait(timeout); err != nil {
+ if err := podInterface.Wait(opts.Timeout); err != nil {
  return nil, err
  }
- if len(exposePorts) > 0 {
+ if len(opts.ExposePorts) > 0 {
  svc := &v1.Service{
  ObjectMeta: metav1.ObjectMeta{
  Name: name,
@@ -396,7 +409,7 @@ func (c *Controller) RunPod(name, image string, command []string, timeout time.D
  Selector: map[string]string{"id": name},
  },
  }
- for _, port := range exposePorts {
+ for _, port := range opts.ExposePorts {
  svc.Spec.Ports = append(svc.Spec.Ports, v1.ServicePort{
  Name: fmt.Sprintf("port%d", port),
  Port: port,
@@ -410,3 +423,48 @@ func (c *Controller) RunPod(name, image string, command []string, timeout time.D
  }
  return podInterface, nil
 }
+
+func generatePodSpec(name, image string, opts RunPodOptions) *v1.Pod {
+ pod := &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{"id": name},
+ },
+ Spec: v1.PodSpec{
+ Containers: []v1.Container{
+ {
+ Name: name,
+ Image: image,
+ ImagePullPolicy: v1.PullIfNotPresent,
+ Command: opts.Command,
+ },
+ },
+ },
+ }
+
+ if opts.NodeName != "" {
+ pod.Spec.NodeSelector = map[string]string{
+ "kubernetes.io/hostname": opts.NodeName,
+ }
+ }
+
+ for n, hpm := range opts.HostPathMounts {
+ name := fmt.Sprintf("vol%d", n)
+ pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
+ Name: name,
+ VolumeSource: v1.VolumeSource{
+ HostPath: &v1.HostPathVolumeSource{
+ Path: hpm.HostPath,
+ },
+ },
+ })
+ pod.Spec.Containers[0].VolumeMounts = append(
+ pod.Spec.Containers[0].VolumeMounts,
+ v1.VolumeMount{
+ Name: name,
+ MountPath: hpm.ContainerPath,
+ })
+ }
+
+ return pod
+}

tests/longevity/checks.go

@@ -20,7 +20,6 @@ import (
  "fmt"
  "regexp"
  "strings"
- "time"
 
  "github.com/Mirantis/virtlet/tests/e2e/framework"
  "github.com/golang/glog"
@@ -93,7 +92,11 @@ func checkInterPodConnectivity(instance *VMInstance) error {
 
 func startNginxPod(controller *framework.Controller) (*framework.PodInterface, error) {
  // Create a Pod to test in-cluster network connectivity
- p, err := controller.RunPod("nginx", framework.NginxImage, nil, time.Minute*4, 80)
+ p, err := controller.RunPod(
+ "nginx", framework.NginxImage,
+ framework.RunPodOptions{
+ ExposePorts: []int32{80},
+ })
  if err != nil {
  return nil, err
  }