From 426b0df8d88d7a1f2e7b2c788ef302e3114751f6 Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Sun, 9 Sep 2018 20:56:02 -0700 Subject: [PATCH] e2e tests for Fleet Scaling and Updates Also includes some functionality for making e2e tests with fleets a bit easier as well. --- build/Makefile | 2 +- examples/simple-udp/Makefile | 2 +- examples/simple-udp/fleet.yaml | 2 +- examples/simple-udp/gameserver.yaml | 2 +- examples/simple-udp/gameserverset.yaml | 2 +- examples/simple-udp/main.go | 11 ++ test/e2e/fleet_test.go | 143 ++++++++++++++++++++++++- test/e2e/framework/framework.go | 70 +++++++++++- test/e2e/main_test.go | 4 +- 9 files changed, 223 insertions(+), 15 deletions(-) diff --git a/build/Makefile b/build/Makefile index 292d158f15..63af3bede2 100644 --- a/build/Makefile +++ b/build/Makefile @@ -47,7 +47,7 @@ GCP_CLUSTER_ZONE ?= us-west1-c MINIKUBE_PROFILE ?= agones # Game Server image to use while doing end-to-end tests -GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.3 +GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.4 # Directory that this Makefile is in. mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) diff --git a/examples/simple-udp/Makefile b/examples/simple-udp/Makefile index d93f3ea026..5880cf8c84 100644 --- a/examples/simple-udp/Makefile +++ b/examples/simple-udp/Makefile @@ -27,7 +27,7 @@ REPOSITORY = gcr.io/agones-images mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) project_path := $(dir $(mkfile_path)) -server_tag = $(REPOSITORY)/udp-server:0.3 +server_tag = $(REPOSITORY)/udp-server:0.4 root_path = $(realpath $(project_path)/../..) # _____ _ diff --git a/examples/simple-udp/fleet.yaml b/examples/simple-udp/fleet.yaml index 413f50d281..f145f9e50d 100644 --- a/examples/simple-udp/fleet.yaml +++ b/examples/simple-udp/fleet.yaml @@ -31,4 +31,4 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.3 \ No newline at end of file + image: gcr.io/agones-images/udp-server:0.4 \ No newline at end of file diff --git a/examples/simple-udp/gameserver.yaml b/examples/simple-udp/gameserver.yaml index ed110b5ea4..019e1357cf 100644 --- a/examples/simple-udp/gameserver.yaml +++ b/examples/simple-udp/gameserver.yaml @@ -25,4 +25,4 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.3 \ No newline at end of file + image: gcr.io/agones-images/udp-server:0.4 \ No newline at end of file diff --git a/examples/simple-udp/gameserverset.yaml b/examples/simple-udp/gameserverset.yaml index 54d5e1c88e..c0f47a0f85 100644 --- a/examples/simple-udp/gameserverset.yaml +++ b/examples/simple-udp/gameserverset.yaml @@ -31,4 +31,4 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.2 \ No newline at end of file + image: gcr.io/agones-images/udp-server:0.4 \ No newline at end of file diff --git a/examples/simple-udp/main.go b/examples/simple-udp/main.go index a767a3166d..56f89d7c2f 100644 --- a/examples/simple-udp/main.go +++ b/examples/simple-udp/main.go @@ -26,12 +26,15 @@ import ( "time" coresdk "agones.dev/agones/pkg/sdk" + "agones.dev/agones/pkg/util/signals" "agones.dev/agones/sdks/go" ) // main starts a UDP server that received 1024 byte sized packets at at time // converts the bytes to a string, and logs the output func main() { + go doSignal() + port := flag.String("port", "7654", "The port to listen to udp traffic on") flag.Parse() if ep := os.Getenv("PORT"); ep != "" { @@ -65,6 +68,14 @@ func main() { readWriteLoop(conn, stop, s) } +// doSignal shutsdown on SIGTERM/SIGKILL +func doSignal() { + stop := signals.NewStopChannel() + <-stop + log.Println("Exit signal received. Shutting down.") + os.Exit(0) +} + func readWriteLoop(conn net.PacketConn, stop chan struct{}, s *sdk.SDK) { b := make([]byte, 1024) for { diff --git a/test/e2e/fleet_test.go b/test/e2e/fleet_test.go index dd69ca42f5..89ead633c8 100644 --- a/test/e2e/fleet_test.go +++ b/test/e2e/fleet_test.go @@ -15,20 +15,30 @@ package e2e import ( + "fmt" "testing" + "time" "agones.dev/agones/pkg/apis/stable/v1alpha1" + e2e "agones.dev/agones/test/e2e/framework" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" ) func TestCreateFleetAndAllocate(t *testing.T) { t.Parallel() - flt, err := framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Create(defaultFleet()) - assert.Nil(t, err) + fleets := framework.AgonesClient.StableV1alpha1().Fleets(defaultNs) + flt, err := fleets.Create(defaultFleet()) + if assert.Nil(t, err) { + defer fleets.Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck + } - err = framework.WaitForFleetReady(flt) + err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(flt.Spec.Replicas)) assert.Nil(t, err, "fleet not ready") fa := &v1alpha1.FleetAllocation{ @@ -43,6 +53,133 @@ func TestCreateFleetAndAllocate(t *testing.T) { assert.Equal(t, v1alpha1.Allocated, fa.Status.GameServer.Status.State) } +func TestScaleFleetUpAndDownWithAllocation(t *testing.T) { + t.Parallel() + alpha1 := framework.AgonesClient.StableV1alpha1() + + flt := defaultFleet() + flt.Spec.Replicas = 1 + flt, err := alpha1.Fleets(defaultNs).Create(flt) + if assert.Nil(t, err) { + defer alpha1.Fleets(defaultNs).Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck + } + + assert.Equal(t, int32(1), flt.Spec.Replicas) + + err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(flt.Spec.Replicas)) + assert.Nil(t, err, "fleet not ready") + + // scale up + flt, err = scaleFleet(flt, 3) + assert.Nil(t, err) + assert.Equal(t, int32(3), flt.Spec.Replicas) + + err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(flt.Spec.Replicas)) + assert.Nil(t, err) + + // get an allocation + fa := &v1alpha1.FleetAllocation{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "allocation-", Namespace: defaultNs}, + Spec: v1alpha1.FleetAllocationSpec{ + FleetName: flt.ObjectMeta.Name, + }, + } + + fa, err = alpha1.FleetAllocations(defaultNs).Create(fa) + assert.Nil(t, err) + assert.Equal(t, v1alpha1.Allocated, fa.Status.GameServer.Status.State) + err = framework.WaitForFleetCondition(flt, func(fleet *v1alpha1.Fleet) bool { + return fleet.Status.AllocatedReplicas == 1 + }) + assert.Nil(t, err) + + // scale down, with allocation + flt, err = scaleFleet(flt, 1) + assert.Nil(t, err) + err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(0)) + assert.Nil(t, err) + + // remove allocation + gp := int64(1) + err = alpha1.GameServers(defaultNs).Delete(fa.Status.GameServer.ObjectMeta.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp}) + assert.Nil(t, err) + err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(1)) + assert.Nil(t, err) + + err = framework.WaitForFleetCondition(flt, func(fleet *v1alpha1.Fleet) bool { + return fleet.Status.AllocatedReplicas == 0 + }) + assert.Nil(t, err) +} + +func TestFleetUpdates(t *testing.T) { + t.Parallel() + + key := "test-state" + fixtures := map[string]func() *v1alpha1.Fleet{ + "recreate": func() *v1alpha1.Fleet { + flt := defaultFleet() + flt.Spec.Strategy.Type = v1.RecreateDeploymentStrategyType + return flt + }, + "rolling": func() *v1alpha1.Fleet { + flt := defaultFleet() + flt.Spec.Strategy.Type = v1.RollingUpdateDeploymentStrategyType + return flt + }, + } + + for k, v := range fixtures { + t.Run(k, func(t *testing.T) { + alpha1 := framework.AgonesClient.StableV1alpha1() + + flt := v() + flt.Spec.Template.ObjectMeta.Annotations = map[string]string{key: "red"} + flt, err := alpha1.Fleets(defaultNs).Create(flt) + if assert.Nil(t, err) { + defer alpha1.Fleets(defaultNs).Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck + } + + err = framework.WaitForFleetGameServersCondition(flt, func(gs v1alpha1.GameServer) bool { + return gs.ObjectMeta.Annotations[key] == "red" + }) + assert.Nil(t, err) + + // if the generation has been updated, it's time to try again. + err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + flt, err = framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Get(flt.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + fltCopy := flt.DeepCopy() + fltCopy.Spec.Template.ObjectMeta.Annotations[key] = "green" + _, err = framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Update(fltCopy) + if err != nil { + logrus.WithError(err).Warn("Could not update fleet, trying again") + return false, nil + } + + return true, nil + }) + assert.Nil(t, err) + + err = framework.WaitForFleetGameServersCondition(flt, func(gs v1alpha1.GameServer) bool { + return gs.ObjectMeta.Annotations[key] == "green" + }) + assert.Nil(t, err) + }) + } +} + +// scaleFleet creates a patch to apply to a Fleet. +// easier for testing, as it removes object generational issues. +func scaleFleet(f *v1alpha1.Fleet, scale int32) (*v1alpha1.Fleet, error) { + patch := fmt.Sprintf(`[{ "op": "replace", "path": "/spec/replicas", "value": %d }]`, scale) + logrus.WithField("fleet", f.ObjectMeta.Name).WithField("scale", scale).WithField("patch", patch).Info("Scaling fleet") + + return framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Patch(f.ObjectMeta.Name, types.JSONPatchType, []byte(patch)) +} + // defaultFleet returns a default fleet configuration func defaultFleet() *v1alpha1.Fleet { gs := defaultGameServer() diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index b60d938602..780cfcbec1 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -21,6 +21,8 @@ import ( "net" "time" + "k8s.io/apimachinery/pkg/labels" + "agones.dev/agones/pkg/apis/stable/v1alpha1" "agones.dev/agones/pkg/client/clientset/versioned" "github.com/pkg/errors" @@ -110,21 +112,79 @@ func (f *Framework) WaitForGameServerState(gs *v1alpha1.GameServer, state v1alph return readyGs, nil } -// WaitForFleetReady waits for the Fleet to count all the GameServers in it as Ready -func (f *Framework) WaitForFleetReady(flt *v1alpha1.Fleet) error { - err := wait.PollImmediate(2*time.Second, 30*time.Second, func() (bool, error) { +// WaitForFleetCondition waits for the Fleet to be in a specific condition +func (f *Framework) WaitForFleetCondition(flt *v1alpha1.Fleet, condition func(fleet *v1alpha1.Fleet) bool) error { + err := wait.PollImmediate(2*time.Second, 120*time.Second, func() (bool, error) { fleet, err := f.AgonesClient.StableV1alpha1().Fleets(flt.ObjectMeta.Namespace).Get(flt.ObjectMeta.Name, metav1.GetOptions{}) if err != nil { return true, err } - return fleet.Status.ReadyReplicas == fleet.Spec.Replicas, nil + return condition(fleet), nil }) return err } -// CleanUp Delete all agones resources in a given namespace +// ListGameServersFromFleet lists GameServers from a particular fleet +func (f *Framework) ListGameServersFromFleet(flt *v1alpha1.Fleet) ([]v1alpha1.GameServer, error) { + var results []v1alpha1.GameServer + + opts := metav1.ListOptions{LabelSelector: labels.Set{v1alpha1.FleetGameServerSetLabel: flt.ObjectMeta.Name}.String()} + gsSetList, err := f.AgonesClient.StableV1alpha1().GameServerSets(flt.ObjectMeta.Namespace).List(opts) + if err != nil { + return results, err + } + + for _, gsSet := range gsSetList.Items { + opts := metav1.ListOptions{LabelSelector: labels.Set{v1alpha1.GameServerSetGameServerLabel: gsSet.ObjectMeta.Name}.String()} + gsList, err := f.AgonesClient.StableV1alpha1().GameServers(flt.ObjectMeta.Namespace).List(opts) + if err != nil { + return results, err + } + + results = append(results, gsList.Items...) + } + + return results, nil +} + +// FleetReadyCountCondition checks the ready count in a fleet +func FleetReadyCount(amount int32) func(fleet *v1alpha1.Fleet) bool { + return func(fleet *v1alpha1.Fleet) bool { + return fleet.Status.ReadyReplicas == amount + } +} + +// WaitForFleetGameServersCondition wait for all GameServers for a given +// fleet to match the spec.replicas and match a a condition +func (f *Framework) WaitForFleetGameServersCondition(flt *v1alpha1.Fleet, cond func(server v1alpha1.GameServer) bool) error { + return wait.Poll(2*time.Second, 5*time.Minute, func() (done bool, err error) { + gsList, err := f.ListGameServersFromFleet(flt) + if err != nil { + return false, err + } + + if int32(len(gsList)) != flt.Spec.Replicas { + return false, nil + } + + if err != nil { + return false, err + } + + for _, gs := range gsList { + if !cond(gs) { + return false, nil + } + } + + return true, nil + }) +} + +// CleanUp Delete all Agones resources in a given namespace func (f *Framework) CleanUp(ns string) error { + logrus.Info("Done. Cleaning up now.") err := f.AgonesClient.StableV1alpha1().Fleets(ns).DeleteCollection(&metav1.DeleteOptions{}, metav1.ListOptions{}) if err != nil { return err diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 31d3beaa75..e0d7b4efa6 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -33,8 +33,8 @@ func TestMain(m *testing.M) { usr, _ := user.Current() kubeconfig := flag.String("kubeconfig", filepath.Join(usr.HomeDir, "/.kube/config"), "kube config path, e.g. $HOME/.kube/config") - gsimage := flag.String("gameserver-image", "gcr.io/agones-images/udp-server:0.3", - "gameserver image to use for those tests, gcr.io/agones-images/udp-server:0.3") + gsimage := flag.String("gameserver-image", "gcr.io/agones-images/udp-server:0.4", + "gameserver image to use for those tests, gcr.io/agones-images/udp-server:0.4") flag.Parse()