diff --git a/build/cmd.sh b/build/cmd.sh index b1a2cb6d4..4ff44f40e 100755 --- a/build/cmd.sh +++ b/build/cmd.sh @@ -444,6 +444,7 @@ function build_internal { go build -i -o "${project_dir}/_output/vmwrapper" ./cmd/vmwrapper go build -i -o "${project_dir}/_output/flexvolume_driver" ./cmd/flexvolume_driver go test -i -c -o "${project_dir}/_output/virtlet-e2e-tests" ./tests/e2e + go build -i -o "${project_dir}/_output/virtlet-longevity-tests" -ldflags "${ldflags}" ./cmd/longevity } function release_description { diff --git a/cmd/longevity/longevity.go b/cmd/longevity/longevity.go new file mode 100644 index 000000000..09a77dfb5 --- /dev/null +++ b/cmd/longevity/longevity.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + "github.com/Mirantis/virtlet/tests/longevity" + "github.com/golang/glog" +) + +func main() { + baseTest := flag.Bool("base", false, "Run base longevity tests") + stressTest := flag.Bool("stress", false, "Run longevity stress tests") + + flag.Parse() + + glog.Infof("Starting Virtlet longevity tests...") + controller, err := framework.NewController("") + if err != nil { + glog.Fatal(err) + } + + instances := []*longevity.VMInstance{} + + if *baseTest { + instances = append(instances, longevity.GetBaseTests(controller)...) + } + if *stressTest { + instances = append(instances, longevity.GetStressTests(controller)...) + } + + err = longevity.Run(controller, instances) + if err != nil { + glog.Fatal(err) + } + controller.Finalize() +} diff --git a/glide.lock b/glide.lock index 29cf2a80c..b9d8f8425 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,8 @@ -hash: 56045aeed24fa36a2b560f87ff6a4af024361a241066d81b5173b885659060f4 -updated: 2018-06-19T21:10:27.123795189+03:00 +hash: b30bbdd1f282c28a740a7fc85d9090ffab31476f58d507fcddbd92e3c1ab1ff8 +updated: 2018-06-29T00:59:01.939551306+02:00 imports: - name: github.com/boltdb/bolt - version: 583e8937c61f1af6513608ccc75c97b6abdf4ff9 + version: fd01fc79c553a8e99d512a07e8e0c63d4a3ccfc5 - name: github.com/containernetworking/cni version: 137b4975ecab6e1f0c24c1e3c228a50a3cfba75e subpackages: @@ -16,7 +16,7 @@ imports: - name: github.com/coreos/go-systemd version: 4484981625c1a6a2ecb40a390fcb6a9bcfee76e3 - name: github.com/cpuguy83/go-md2man - version: 48d8747a2ca13185e7cc8efe6e9fc196a83f71a5 + version: 691ee98543af2f262f35fbb54bdd42f00b9b9cc5 subpackages: - md2man - name: github.com/davecgh/go-spew @@ -168,7 +168,7 @@ imports: - reporters/stenographer - types - name: github.com/onsi/gomega - version: 2152b45fa28a361beba9aab0885972323a444e28 + version: 62bff4df71bdbc266561a0caee19f0594b17c240 subpackages: - format - internal/assertion @@ -226,6 +226,9 @@ imports: subpackages: - bpf - context + - html + - html/atom + - html/charset - http2 - http2/hpack - idna @@ -249,6 +252,20 @@ imports: - name: golang.org/x/text version: b19bf474d317b857955b12035d2c5acb57ce8b01 subpackages: + - encoding + - encoding/charmap + - encoding/htmlindex + - encoding/internal + - encoding/internal/identifier + - encoding/japanese + - encoding/korean + - encoding/simplifiedchinese + - encoding/traditionalchinese + - encoding/unicode + - internal/tag + - internal/utf8internal + - language + - runes - secure/bidirule - transform - unicode/bidi diff --git a/glide.yaml b/glide.yaml index 853321e44..72faeaa81 100644 --- a/glide.yaml +++ b/glide.yaml @@ -35,6 +35,7 @@ import: version: bcac9884e7502bb2b474c0339d889cb981a2f27f - package: github.com/onsi/ginkgo - package: github.com/onsi/gomega + version: v1.4.0 - package: golang.org/x/sync subpackages: - syncmap @@ -55,6 +56,8 @@ import: version: edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c subpackages: - reference +- package: github.com/docker/engine-api + version: dea108d3aa0c67d7162a3fd8aa65f38a430019fd - package: github.com/opencontainers/go-digest version: a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb - package: github.com/spf13/pflag diff --git a/tests/e2e/basic_test.go b/tests/e2e/basic_test.go index ee1fc7143..e05a83b15 100644 --- a/tests/e2e/basic_test.go +++ b/tests/e2e/basic_test.go @@ -38,7 +38,7 @@ var _ = Describe("Virtlet [Basic cirros tests]", func() { BeforeAll(func() { vm = controller.VM("cirros-vm") - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) var err error vmPod, err = vm.Pod() Expect(err).NotTo(HaveOccurred()) @@ -182,7 +182,7 @@ var _ = Describe("Virtlet [Disruptive]", func() { Expect(err).NotTo(HaveOccurred()) vm = controller.VM("cirros-vm") - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) }) diff --git a/tests/e2e/ceph_test.go b/tests/e2e/ceph_test.go index 5fa09fb91..a90cba5b7 100644 --- a/tests/e2e/ceph_test.go +++ b/tests/e2e/ceph_test.go @@ -65,7 +65,7 @@ var _ = Describe("Ceph volumes tests", func() { }) } - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) var err error _, err = vm.Pod() Expect(err).NotTo(HaveOccurred()) @@ -141,7 +141,7 @@ var _ = Describe("Ceph volumes tests", func() { }) } - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) _ = do(vm.Pod()).(*framework.PodInterface) }) diff --git a/tests/e2e/cloudinit_test.go b/tests/e2e/cloudinit_test.go index aa5b5b3a7..f01595d88 100644 --- a/tests/e2e/cloudinit_test.go +++ b/tests/e2e/cloudinit_test.go @@ -37,7 +37,7 @@ var _ = Describe("Cloud-init related tests", func() { Name: "cm-ssh-key-impl", }, Data: map[string]string{ - "authorized_keys": sshPublicKey, + "authorized_keys": SshPublicKey, }, } _, err := controller.ConfigMaps().Create(cm) @@ -46,7 +46,7 @@ var _ = Describe("Cloud-init related tests", func() { vm = controller.VM("ssh-from-cm-impl") Expect(vm.Create(VMOptions{ SSHKeySource: "configmap/cm-ssh-key-impl", - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) AfterAll(func() { @@ -68,7 +68,7 @@ var _ = Describe("Cloud-init related tests", func() { Name: "cm-ssh-key-expl", }, Data: map[string]string{ - "myKey": sshPublicKey, + "myKey": SshPublicKey, }, } _, err := controller.ConfigMaps().Create(cm) @@ -77,7 +77,7 @@ var _ = Describe("Cloud-init related tests", func() { vm = controller.VM("ssh-from-cm-expl") Expect(vm.Create(VMOptions{ SSHKeySource: "configmap/cm-ssh-key-expl/myKey", - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) AfterAll(func() { @@ -99,7 +99,7 @@ var _ = Describe("Cloud-init related tests", func() { Name: "secret-ssh-key-impl", }, StringData: map[string]string{ - "authorized_keys": sshPublicKey, + "authorized_keys": SshPublicKey, }, } _, err := controller.Secrets().Create(secret) @@ -108,7 +108,7 @@ var _ = Describe("Cloud-init related tests", func() { vm = controller.VM("ssh-from-secret-impl") Expect(vm.Create(VMOptions{ SSHKeySource: "secret/secret-ssh-key-impl", - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) AfterAll(func() { @@ -130,7 +130,7 @@ var _ = Describe("Cloud-init related tests", func() { Name: "secret-ssh-key-expl", }, StringData: map[string]string{ - "myKey": sshPublicKey, + "myKey": SshPublicKey, }, } _, err := controller.Secrets().Create(secret) @@ -139,7 +139,7 @@ var _ = Describe("Cloud-init related tests", func() { vm = controller.VM("ssh-from-secret-expl") Expect(vm.Create(VMOptions{ SSHKeySource: "secret/secret-ssh-key-expl/myKey", - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) AfterAll(func() { @@ -173,7 +173,7 @@ var _ = Describe("Cloud-init related tests", func() { vm = controller.VM("userdata-cm") Expect(vm.Create(VMOptions{ UserDataSource: "configmap/cm-userdata", - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) AfterAll(func() { @@ -208,7 +208,7 @@ var _ = Describe("Cloud-init related tests", func() { vm = controller.VM("userdata-secret") Expect(vm.Create(VMOptions{ UserDataSource: "secret/secret-userdata", - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) AfterAll(func() { @@ -245,7 +245,7 @@ var _ = Describe("Cloud-init related tests", func() { Expect(vm.Create(VMOptions{ UserDataSource: "configmap/cm-userdata", UserData: userData, - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) }) AfterAll(func() { diff --git a/tests/e2e/cni_test.go b/tests/e2e/cni_test.go index de292f2b9..fe5e8a21a 100644 --- a/tests/e2e/cni_test.go +++ b/tests/e2e/cni_test.go @@ -7,7 +7,7 @@ You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software +Unless required by Applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and @@ -35,7 +35,7 @@ var _ = Describe("Virtlet CNI", func() { // if network namespace was deleted It("Should delete network namespace when VM is deleted", func() { vm = controller.VM("cirros-vm") - err := vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil) + err := vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil) Expect(err).NotTo(HaveOccurred()) virtletPod, err := vm.VirtletPod() diff --git a/tests/e2e/common.go b/tests/e2e/common.go index 751f42029..d3c4367c9 100644 --- a/tests/e2e/common.go +++ b/tests/e2e/common.go @@ -34,7 +34,7 @@ import ( var ( vmImageLocation = flag.String("image", defaultVMImageLocation, "VM image URL (*without http(s)://*") - sshUser = flag.String("sshuser", defaultSSHUser, "default SSH user for VMs") + sshUser = flag.String("sshuser", DefaultSSHUser, "default SSH user for VMs") includeCloudInitTests = flag.Bool("include-cloud-init-tests", false, "include Cloud-Init tests") includeUnsafeTests = flag.Bool("include-unsafe-tests", false, "include tests that can be unsafe if they're run outside the build container") memoryLimit = flag.Int("memoryLimit", 160, "default VM memory limit (in MiB)") @@ -57,7 +57,7 @@ func waitSSH(vm *framework.VMInterface) framework.Executor { Eventually( func() error { var err error - ssh, err = vm.SSH(*sshUser, sshPrivateKey) + ssh, err = vm.SSH(*sshUser, SshPrivateKey) if err != nil { return err } @@ -139,13 +139,13 @@ func do(value interface{}, extra ...interface{}) interface{} { type VMOptions framework.VMOptions -func (o VMOptions) applyDefaults() framework.VMOptions { +func (o VMOptions) ApplyDefaults() framework.VMOptions { res := framework.VMOptions(o) if res.Image == "" { res.Image = *vmImageLocation } if res.SSHKey == "" && res.SSHKeySource == "" { - res.SSHKey = sshPublicKey + res.SSHKey = SshPublicKey } if res.VCPUCount == 0 { res.VCPUCount = 1 diff --git a/tests/e2e/constants.go b/tests/e2e/constants.go index 7f0bfcb5a..10c3dba6a 100644 --- a/tests/e2e/constants.go +++ b/tests/e2e/constants.go @@ -18,14 +18,14 @@ package e2e const ( defaultVMImageLocation = "download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img" - defaultSSHUser = "cirros" + DefaultSSHUser = "cirros" - sshPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+deP" + + SshPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+deP" + "xdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwW" + "zAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0" + "334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost" - sshPrivateKey = ` + SshPrivateKey = ` -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAmiRHBQ1xCtmW19GS0tRCGBWUWwwHEX7lY6bLUnNA3vPrFdWo rvnXj8XZLg0cKhbQspOnRGZLLIzpAV7a4tNDBAi5Iq+hXB95CjibWyer8t7miXPc diff --git a/tests/e2e/framework/common.go b/tests/e2e/framework/common.go index b6ef03315..5b058d154 100644 --- a/tests/e2e/framework/common.go +++ b/tests/e2e/framework/common.go @@ -67,7 +67,7 @@ func RunSimple(executor Executor, command ...string) (string, error) { if err != nil { if ce, ok := err.(CommandError); ok { if ce.ExitCode != 0 { - return "", fmt.Errorf("command exited with code %d, stderr: %s", ce.ExitCode, strings.TrimSpace(stderr)) + return "", fmt.Errorf("command exited with code %d, stderr: %s", ce.ExitCode, strings.TrimSpace(stderr)+strings.TrimSpace(stdout)) } return strings.TrimSpace(stdout), nil } diff --git a/tests/e2e/hung_vm_test.go b/tests/e2e/hung_vm_test.go index 86c4ff178..5d9aaa8ad 100644 --- a/tests/e2e/hung_vm_test.go +++ b/tests/e2e/hung_vm_test.go @@ -33,7 +33,7 @@ var _ = Describe("Hung VM", func() { BeforeAll(func() { vm = controller.VM("hung-vm") - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) var err error _, err = vm.Pod() Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/image_name_translation_test.go b/tests/e2e/image_name_translation_test.go index 6ebb8fa07..495f81861 100644 --- a/tests/e2e/image_name_translation_test.go +++ b/tests/e2e/image_name_translation_test.go @@ -54,7 +54,7 @@ var _ = Describe("Image URL", func() { It("Can be specified in CRD [Conformance]", func() { vm := controller.VM("cirros-vm-with-remapped-image") - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) _, err := vm.Pod() Expect(err).NotTo(HaveOccurred()) deleteVM(vm) diff --git a/tests/e2e/multi_cni_test.go b/tests/e2e/multi_cni_test.go index 1759f595b..7670a4efd 100644 --- a/tests/e2e/multi_cni_test.go +++ b/tests/e2e/multi_cni_test.go @@ -100,7 +100,7 @@ func makeMultiCNIVM(name string, addCNIAnnotation bool) *multiCNIVM { if addCNIAnnotation { opts.MultiCNI = "calico,flannel" } - Expect(vm.Create(opts.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + Expect(vm.Create(opts.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) vmPod, err := vm.Pod() Expect(err).NotTo(HaveOccurred()) return &multiCNIVM{ diff --git a/tests/e2e/resources_test.go b/tests/e2e/resources_test.go index 01ececcde..3113fbcc8 100644 --- a/tests/e2e/resources_test.go +++ b/tests/e2e/resources_test.go @@ -37,7 +37,7 @@ var _ = Describe("VM resources", func() { vm = controller.VM("vm-resources") Expect(vm.Create(VMOptions{ VCPUCount: 2, - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) do(vm.Pod()) }) diff --git a/tests/e2e/restart_virtlet_test.go b/tests/e2e/restart_virtlet_test.go index 7bb4d7637..dbf37c580 100644 --- a/tests/e2e/restart_virtlet_test.go +++ b/tests/e2e/restart_virtlet_test.go @@ -36,7 +36,7 @@ var _ = Describe("Virtlet restart [Disruptive]", func() { BeforeAll(func() { vm = controller.VM("cirros-vm") - vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil) + vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil) var err error vmPod, err = vm.Pod() Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/stop_qemu_test.go b/tests/e2e/stop_qemu_test.go index 45c64d7cc..8ee138650 100644 --- a/tests/e2e/stop_qemu_test.go +++ b/tests/e2e/stop_qemu_test.go @@ -37,7 +37,7 @@ var _ = Describe("QEMU Process", func() { BeforeAll(func() { vm = controller.VM("kill-qemu-vm") - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) var err error vmPod, err := vm.Pod() Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/virtletctl_test.go b/tests/e2e/virtletctl_test.go index 46a2b3c42..06100dbdb 100644 --- a/tests/e2e/virtletctl_test.go +++ b/tests/e2e/virtletctl_test.go @@ -59,7 +59,7 @@ var _ = Describe("virtletctl", func() { Name: "sshkey", }, Data: map[string]string{ - "authorized_keys": sshPublicKey, + "authorized_keys": SshPublicKey, }, } _, err := controller.ConfigMaps().Create(cm) @@ -68,7 +68,7 @@ var _ = Describe("virtletctl", func() { vm = controller.VM("virtletctl-cirros-vm") Expect(vm.Create(VMOptions{ SSHKeySource: "configmap/sshkey", - }.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, nil)).To(Succeed()) waitSSH(vm) @@ -77,7 +77,7 @@ var _ = Describe("virtletctl", func() { defer tempfile.Close() tempfileName = tempfile.Name() - strippedKey := strings.Replace(sshPrivateKey, "\t", "", -1) + strippedKey := strings.Replace(SshPrivateKey, "\t", "", -1) _, err = tempfile.Write([]byte(strippedKey)) Expect(err).NotTo(HaveOccurred()) Expect(os.Chmod(tempfileName, 0600)).To(Succeed()) diff --git a/tests/e2e/volume_mount_test.go b/tests/e2e/volume_mount_test.go index 20575e1b3..403d91c9f 100644 --- a/tests/e2e/volume_mount_test.go +++ b/tests/e2e/volume_mount_test.go @@ -177,7 +177,7 @@ var _ = Describe("Container volume mounts", func() { }) } - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) }) AfterAll(func() { @@ -236,7 +236,7 @@ var _ = Describe("Container volume mounts", func() { }) } - Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) + Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) }) AfterAll(func() { @@ -276,7 +276,7 @@ func makeVolumeMountVM(nodeName string, podCustomization func(*framework.PodInte // ubuntu image with volumes mounted using cloud-init // userdata 'mounts' section UserDataScript: "@virtlet-mount-script@", - }.applyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) + }.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed()) _, err := vm.Pod() Expect(err).NotTo(HaveOccurred()) return vm diff --git a/tests/longevity/README.md b/tests/longevity/README.md new file mode 100644 index 000000000..d3b0c719a --- /dev/null +++ b/tests/longevity/README.md @@ -0,0 +1,11 @@ +# Longevity tests + +The goal of longevity test how Virtlet behaves after running for a long time. + +To start tests locally run: + +```shell +_output/virtlet-longevity-tests -stress -alsologtostderr +``` + +Tests will run until error occurs or CTRL-C is received diff --git a/tests/longevity/checks.go b/tests/longevity/checks.go new file mode 100644 index 000000000..321900b14 --- /dev/null +++ b/tests/longevity/checks.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package longevity + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + "github.com/golang/glog" +) + +func testVM(instance *VMInstance) error { + err := checkDefaultRoute(instance) + if err != nil { + return fmt.Errorf("Checking default route failed: %v", err) + } + err = checkInternetConnectivity(instance) + if err != nil { + return fmt.Errorf("Checking internet connectivity failed: %v", err) + } + err = checkInterPodConnectivity(instance) + if err != nil { + return fmt.Errorf("Checking inter-pod connectivity failed: %v", err) + } + return nil +} + +func checkDefaultRoute(instance *VMInstance) error { + vmPod, err := instance.vm.Pod() + if err != nil { + return err + } + + glog.V(4).Infof("Should have default route") + out, err := framework.RunSimple(instance.ssh, "ip r") + if err != nil { + return err + } + if !strings.Contains(out, "default via") { + return fmt.Errorf("Should contain `default via` line but it's missing") + } + if !strings.Contains(out, "src "+vmPod.Pod.Status.PodIP) { + return fmt.Errorf("Should contain `src %s` line but it's missing", vmPod.Pod.Status.PodIP) + } + return nil +} + +func checkInternetConnectivity(instance *VMInstance) error { + glog.V(4).Infof("Should have internet connectivity") + output, err := framework.RunSimple(instance.ssh, "ping -c1 8.8.8.8") + if err != nil { + return fmt.Errorf("Error when running command ping -c1 8.8.8.8: %v", err) + } + matched, err := regexp.MatchString("1 .*transmitted, 1 .*received, 0% .*loss", output) + if err != nil { + return fmt.Errorf("Error when running regexp: %v", err) + } + if !matched { + return fmt.Errorf("No internet connectivity. ping output: ```%s```", output) + } + return nil +} + +func checkInterPodConnectivity(instance *VMInstance) error { + glog.V(4).Infof("Should be able to access another k8s endpoint") + cmd := fmt.Sprintf("curl -s --connect-timeout 5 http://nginx.%s.svc.cluster.local", instance.controller.Namespace()) + out, err := framework.RunSimple(instance.ssh, cmd) + if err != nil { + return fmt.Errorf("Error when running curl: %v", err) + } + if !strings.Contains(out, "Thank you for using nginx.") { + return fmt.Errorf("Should contain `Thank you for using nginx.` line but it's missing") + } + return nil +} + +func startNginxPod(controller *framework.Controller) (*framework.PodInterface, error) { + // Create a Pod to test in-cluster network connectivity + p, err := controller.RunPod("nginx", "nginx", nil, time.Minute*4, 80) + if err != nil { + return nil, err + } + + return p, nil +} diff --git a/tests/longevity/runner.go b/tests/longevity/runner.go new file mode 100644 index 000000000..8e3bf7ad0 --- /dev/null +++ b/tests/longevity/runner.go @@ -0,0 +1,123 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package longevity + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + "github.com/golang/glog" +) + +func GetBaseTests(controller *framework.Controller) []*VMInstance { + return []*VMInstance{ + { + name: "cirros-base-test", + controller: controller, + lifetime: time.Duration(24) * time.Hour, + testWaitTime: time.Duration(1) * time.Hour, + }, + } +} + +func GetStressTests(controller *framework.Controller) []*VMInstance { + return []*VMInstance{ + { + name: "cirros-stress-1min", + controller: controller, + lifetime: time.Duration(1) * time.Minute, + testWaitTime: time.Duration(1) * time.Minute, + }, + { + name: "cirros-stress-3min", + controller: controller, + lifetime: time.Duration(3) * time.Minute, + testWaitTime: time.Duration(1) * time.Minute, + }, + { + name: "cirros-stress-5min", + controller: controller, + lifetime: time.Duration(5) * time.Minute, + testWaitTime: time.Duration(1) * time.Minute, + }, + { + name: "cirros-stress-10min", + controller: controller, + lifetime: time.Duration(10) * time.Minute, + testWaitTime: time.Duration(1) * time.Minute, + }, + { + name: "cirros-stress-15min", + controller: controller, + lifetime: time.Duration(15) * time.Minute, + testWaitTime: time.Duration(1) * time.Minute, + }, + } +} + +func Run(controller *framework.Controller, instances []*VMInstance) error { + var err error + errChan := make(chan error) + + exitChan := make(chan os.Signal, 1) + signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + select { + case <-exitChan: + glog.V(4).Infof("CTRL-C received") + cancel() + case <-ctx.Done(): + break + } + }() + + _, err = startNginxPod(controller) + if err != nil { + return fmt.Errorf("Couldn't start nginx pod: %v", err) + } + + for _, instance := range instances { + glog.Infof("Creating `%s` VM...", instance.name) + err = instance.Create() + if err != nil { + return fmt.Errorf("Could not create VM: %v", err) + } + glog.V(4).Infof("Done") + } + for _, instance := range instances { + go instance.Test(ctx, instance, testVM, errChan) + } + + for { + select { + case err = <-errChan: + glog.V(4).Infof("Received error: %v", err) + cancel() + return err + case <-ctx.Done(): + glog.Infof("Finishing testing...") + return nil + } + } + return nil +} diff --git a/tests/longevity/tools.go b/tests/longevity/tools.go new file mode 100644 index 000000000..b531e78d4 --- /dev/null +++ b/tests/longevity/tools.go @@ -0,0 +1,31 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package longevity + +import ( + "os" + "strconv" +) + +func getEnvInt(env string, defaultVal int) int { + envVal := os.Getenv(env) + val, err := strconv.Atoi(envVal) + if err == nil { + return val + } + return defaultVal +} diff --git a/tests/longevity/vmcontoller.go b/tests/longevity/vmcontoller.go new file mode 100644 index 000000000..560bd1c7d --- /dev/null +++ b/tests/longevity/vmcontoller.go @@ -0,0 +1,128 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package longevity + +import ( + "context" + "fmt" + "time" + + "github.com/Mirantis/virtlet/tests/e2e" + "github.com/Mirantis/virtlet/tests/e2e/framework" + "github.com/golang/glog" +) + +type VMInstance struct { + name string + ssh framework.Executor + vm *framework.VMInterface + controller *framework.Controller + testWaitTime time.Duration + lifetime time.Duration + failures int + testTicker *time.Ticker + lifetimeTicker *time.Ticker +} + +func (i *VMInstance) Test(ctx context.Context, instance *VMInstance, testFunc func(*VMInstance) error, errCh chan error) { + defer instance.Stop() + var err error + for { + select { + case <-ctx.Done(): + glog.V(3).Infof("Testing '%s' VM stopped", instance.name) + return + case <-instance.testTicker.C: + glog.V(3).Infof("Testing '%s' VM...", instance.name) + err = testFunc(instance) + if err != nil { + glog.V(4).Infof("Test function failed with: %v, instance: %+v", err, instance) + instance.failures++ + // there are mostly network tests so we allow one glitch + if instance.failures > 1 { + errCh <- fmt.Errorf("Testing VM %s failed: %v", instance.name, err) + return + } + } + case <-instance.lifetimeTicker.C: + glog.V(4).Infof("Recreating VM: %s", instance.name) + err = instance.ReCreate() + if err != nil { + glog.V(4).Infof("Recreating VM %s failed: %v", instance.name, err) + errCh <- fmt.Errorf("Failed to recreate VM %s: %v", instance.name, err) + return + } + } + } +} + +func (i *VMInstance) Create() error { + var err error + + i.vm = i.controller.VM(i.name) + err = i.vm.Create(e2e.VMOptions{}.ApplyDefaults(), time.Minute*5, nil) + if err != nil { + return err + } + + err, i.ssh = waitSSH(i.vm) + if err != nil { + return err + } + i.lifetimeTicker = time.NewTicker(i.lifetime) + i.testTicker = time.NewTicker(i.testWaitTime) + return nil +} + +func (i *VMInstance) Delete() error { + err := i.vm.Delete(30 * time.Second) + if err != nil { + return err + } + return nil +} + +func (i *VMInstance) ReCreate() error { + err := i.Delete() + if err != nil { + return err + } + return i.Create() +} + +func waitSSH(vm *framework.VMInterface) (error, framework.Executor) { + var err error + var ssh framework.Executor + for i := 0; i < 60*5; i += 3 { + time.Sleep(3 * time.Second) + ssh, err = vm.SSH(e2e.DefaultSSHUser, e2e.SshPrivateKey) + if err != nil { + continue + } + _, err = framework.RunSimple(ssh) + if err != nil { + continue + } + return nil, ssh + } + return fmt.Errorf("Timeout waiting for ssh connection to vm: %v", vm), nil +} + +func (i *VMInstance) Stop() { + i.lifetimeTicker.Stop() + i.testTicker.Stop() +}