e2e tests for Fleet Scaling and Updates

Also includes some functionality for making
e2e tests with fleets a bit easier as well.

build/Makefile

@@ -47,7 +47,7 @@ GCP_CLUSTER_ZONE ?= us-west1-c
 MINIKUBE_PROFILE ?= agones
 
 # Game Server image to use while doing end-to-end tests
-GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.3
+GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.4
 
 # Directory that this Makefile is in.
 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))

examples/simple-udp/Makefile

@@ -27,7 +27,7 @@ REPOSITORY = gcr.io/agones-images
 
 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
 project_path := $(dir $(mkfile_path))
-server_tag = $(REPOSITORY)/udp-server:0.3
+server_tag = $(REPOSITORY)/udp-server:0.4
 root_path = $(realpath $(project_path)/../..)
 
 #   _____                    _

examples/simple-udp/fleet.yaml

@@ -31,4 +31,4 @@ spec:
         spec:
           containers:
           - name: simple-udp
-            image: gcr.io/agones-images/udp-server:0.3
+            image: gcr.io/agones-images/udp-server:0.4

examples/simple-udp/gameserver.yaml

@@ -25,4 +25,4 @@ spec:
     spec:
       containers:
       - name: simple-udp
-        image: gcr.io/agones-images/udp-server:0.3
+        image: gcr.io/agones-images/udp-server:0.4

examples/simple-udp/gameserverset.yaml

@@ -31,4 +31,4 @@ spec:
         spec:
           containers:
           - name: simple-udp
-            image: gcr.io/agones-images/udp-server:0.2
+            image: gcr.io/agones-images/udp-server:0.4

examples/simple-udp/main.go

@@ -26,12 +26,15 @@ import (
 	"time"
 
 	coresdk "agones.dev/agones/pkg/sdk"
+	"agones.dev/agones/pkg/util/signals"
 	"agones.dev/agones/sdks/go"
 )
 
 // main starts a UDP server that received 1024 byte sized packets at at time
 // converts the bytes to a string, and logs the output
 func main() {
+	go doSignal()
+
 	port := flag.String("port", "7654", "The port to listen to udp traffic on")
 	flag.Parse()
 	if ep := os.Getenv("PORT"); ep != "" {
@@ -65,6 +68,14 @@ func main() {
 	readWriteLoop(conn, stop, s)
 }
 
+// doSignal shutsdown on SIGTERM/SIGKILL
+func doSignal() {
+	stop := signals.NewStopChannel()
+	<-stop
+	log.Println("Exit signal received. Shutting down.")
+	os.Exit(0)
+}
+
 func readWriteLoop(conn net.PacketConn, stop chan struct{}, s *sdk.SDK) {
 	b := make([]byte, 1024)
 	for {

test/e2e/fleet_test.go

@@ -15,20 +15,30 @@
 package e2e
 
 import (
+	"fmt"
 	"testing"
+	"time"
 
 	"agones.dev/agones/pkg/apis/stable/v1alpha1"
+	e2e "agones.dev/agones/test/e2e/framework"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 func TestCreateFleetAndAllocate(t *testing.T) {
 	t.Parallel()
 
-	flt, err := framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Create(defaultFleet())
-	assert.Nil(t, err)
+	fleets := framework.AgonesClient.StableV1alpha1().Fleets(defaultNs)
+	flt, err := fleets.Create(defaultFleet())
+	if assert.Nil(t, err) {
+		defer fleets.Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck
+	}
 
-	err = framework.WaitForFleetReady(flt)
+	err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(flt.Spec.Replicas))
 	assert.Nil(t, err, "fleet not ready")
 
 	fa := &v1alpha1.FleetAllocation{
@@ -43,6 +53,133 @@ func TestCreateFleetAndAllocate(t *testing.T) {
 	assert.Equal(t, v1alpha1.Allocated, fa.Status.GameServer.Status.State)
 }
 
+func TestScaleFleetUpAndDownWithAllocation(t *testing.T) {
+	t.Parallel()
+	alpha1 := framework.AgonesClient.StableV1alpha1()
+
+	flt := defaultFleet()
+	flt.Spec.Replicas = 1
+	flt, err := alpha1.Fleets(defaultNs).Create(flt)
+	if assert.Nil(t, err) {
+		defer alpha1.Fleets(defaultNs).Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck
+	}
+
+	assert.Equal(t, int32(1), flt.Spec.Replicas)
+
+	err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(flt.Spec.Replicas))
+	assert.Nil(t, err, "fleet not ready")
+
+	// scale up
+	flt, err = scaleFleet(flt, 3)
+	assert.Nil(t, err)
+	assert.Equal(t, int32(3), flt.Spec.Replicas)
+
+	err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(flt.Spec.Replicas))
+	assert.Nil(t, err)
+
+	// get an allocation
+	fa := &v1alpha1.FleetAllocation{
+		ObjectMeta: metav1.ObjectMeta{GenerateName: "allocation-", Namespace: defaultNs},
+		Spec: v1alpha1.FleetAllocationSpec{
+			FleetName: flt.ObjectMeta.Name,
+		},
+	}
+
+	fa, err = alpha1.FleetAllocations(defaultNs).Create(fa)
+	assert.Nil(t, err)
+	assert.Equal(t, v1alpha1.Allocated, fa.Status.GameServer.Status.State)
+	err = framework.WaitForFleetCondition(flt, func(fleet *v1alpha1.Fleet) bool {
+		return fleet.Status.AllocatedReplicas == 1
+	})
+	assert.Nil(t, err)
+
+	// scale down, with allocation
+	flt, err = scaleFleet(flt, 1)
+	assert.Nil(t, err)
+	err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(0))
+	assert.Nil(t, err)
+
+	// remove allocation
+	gp := int64(1)
+	err = alpha1.GameServers(defaultNs).Delete(fa.Status.GameServer.ObjectMeta.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})
+	assert.Nil(t, err)
+	err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(1))
+	assert.Nil(t, err)
+
+	err = framework.WaitForFleetCondition(flt, func(fleet *v1alpha1.Fleet) bool {
+		return fleet.Status.AllocatedReplicas == 0
+	})
+	assert.Nil(t, err)
+}
+
+func TestFleetUpdates(t *testing.T) {
+	t.Parallel()
+
+	key := "test-state"
+	fixtures := map[string]func() *v1alpha1.Fleet{
+		"recreate": func() *v1alpha1.Fleet {
+			flt := defaultFleet()
+			flt.Spec.Strategy.Type = v1.RecreateDeploymentStrategyType
+			return flt
+		},
+		"rolling": func() *v1alpha1.Fleet {
+			flt := defaultFleet()
+			flt.Spec.Strategy.Type = v1.RollingUpdateDeploymentStrategyType
+			return flt
+		},
+	}
+
+	for k, v := range fixtures {
+		t.Run(k, func(t *testing.T) {
+			alpha1 := framework.AgonesClient.StableV1alpha1()
+
+			flt := v()
+			flt.Spec.Template.ObjectMeta.Annotations = map[string]string{key: "red"}
+			flt, err := alpha1.Fleets(defaultNs).Create(flt)
+			if assert.Nil(t, err) {
+				defer alpha1.Fleets(defaultNs).Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck
+			}
+
+			err = framework.WaitForFleetGameServersCondition(flt, func(gs v1alpha1.GameServer) bool {
+				return gs.ObjectMeta.Annotations[key] == "red"
+			})
+			assert.Nil(t, err)
+
+			// if the generation has been updated, it's time to try again.
+			err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
+				flt, err = framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Get(flt.ObjectMeta.Name, metav1.GetOptions{})
+				if err != nil {
+					return false, err
+				}
+				fltCopy := flt.DeepCopy()
+				fltCopy.Spec.Template.ObjectMeta.Annotations[key] = "green"
+				_, err = framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Update(fltCopy)
+				if err != nil {
+					logrus.WithError(err).Warn("Could not update fleet, trying again")
+					return false, nil
+				}
+
+				return true, nil
+			})
+			assert.Nil(t, err)
+
+			err = framework.WaitForFleetGameServersCondition(flt, func(gs v1alpha1.GameServer) bool {
+				return gs.ObjectMeta.Annotations[key] == "green"
+			})
+			assert.Nil(t, err)
+		})
+	}
+}
+
+// scaleFleet creates a patch to apply to a Fleet.
+// easier for testing, as it removes object generational issues.
+func scaleFleet(f *v1alpha1.Fleet, scale int32) (*v1alpha1.Fleet, error) {
+	patch := fmt.Sprintf(`[{ "op": "replace", "path": "/spec/replicas", "value": %d }]`, scale)
+	logrus.WithField("fleet", f.ObjectMeta.Name).WithField("scale", scale).WithField("patch", patch).Info("Scaling fleet")
+
+	return framework.AgonesClient.StableV1alpha1().Fleets(defaultNs).Patch(f.ObjectMeta.Name, types.JSONPatchType, []byte(patch))
+}
+
 // defaultFleet returns a default fleet configuration
 func defaultFleet() *v1alpha1.Fleet {
 	gs := defaultGameServer()

test/e2e/framework/framework.go

@@ -21,6 +21,8 @@ import (
 	"net"
 	"time"
 
+	"k8s.io/apimachinery/pkg/labels"
+
 	"agones.dev/agones/pkg/apis/stable/v1alpha1"
 	"agones.dev/agones/pkg/client/clientset/versioned"
 	"github.com/pkg/errors"
@@ -110,21 +112,79 @@ func (f *Framework) WaitForGameServerState(gs *v1alpha1.GameServer, state v1alph
 	return readyGs, nil
 }
 
-// WaitForFleetReady waits for the Fleet to count all the GameServers in it as Ready
-func (f *Framework) WaitForFleetReady(flt *v1alpha1.Fleet) error {
-	err := wait.PollImmediate(2*time.Second, 30*time.Second, func() (bool, error) {
+// WaitForFleetCondition waits for the Fleet to be in a specific condition
+func (f *Framework) WaitForFleetCondition(flt *v1alpha1.Fleet, condition func(fleet *v1alpha1.Fleet) bool) error {
+	err := wait.PollImmediate(2*time.Second, 120*time.Second, func() (bool, error) {
 		fleet, err := f.AgonesClient.StableV1alpha1().Fleets(flt.ObjectMeta.Namespace).Get(flt.ObjectMeta.Name, metav1.GetOptions{})
 		if err != nil {
 			return true, err
 		}
 
-		return fleet.Status.ReadyReplicas == fleet.Spec.Replicas, nil
+		return condition(fleet), nil
 	})
 	return err
 }
 
-// CleanUp Delete all agones resources in a given namespace
+// ListGameServersFromFleet lists GameServers from a particular fleet
+func (f *Framework) ListGameServersFromFleet(flt *v1alpha1.Fleet) ([]v1alpha1.GameServer, error) {
+	var results []v1alpha1.GameServer
+
+	opts := metav1.ListOptions{LabelSelector: labels.Set{v1alpha1.FleetGameServerSetLabel: flt.ObjectMeta.Name}.String()}
+	gsSetList, err := f.AgonesClient.StableV1alpha1().GameServerSets(flt.ObjectMeta.Namespace).List(opts)
+	if err != nil {
+		return results, err
+	}
+
+	for _, gsSet := range gsSetList.Items {
+		opts := metav1.ListOptions{LabelSelector: labels.Set{v1alpha1.GameServerSetGameServerLabel: gsSet.ObjectMeta.Name}.String()}
+		gsList, err := f.AgonesClient.StableV1alpha1().GameServers(flt.ObjectMeta.Namespace).List(opts)
+		if err != nil {
+			return results, err
+		}
+
+		results = append(results, gsList.Items...)
+	}
+
+	return results, nil
+}
+
+// FleetReadyCountCondition checks the ready count in a fleet
+func FleetReadyCount(amount int32) func(fleet *v1alpha1.Fleet) bool {
+	return func(fleet *v1alpha1.Fleet) bool {
+		return fleet.Status.ReadyReplicas == amount
+	}
+}
+
+// WaitForFleetGameServersCondition wait for all GameServers for a given
+// fleet to match the spec.replicas and match a a condition
+func (f *Framework) WaitForFleetGameServersCondition(flt *v1alpha1.Fleet, cond func(server v1alpha1.GameServer) bool) error {
+	return wait.Poll(2*time.Second, 5*time.Minute, func() (done bool, err error) {
+		gsList, err := f.ListGameServersFromFleet(flt)
+		if err != nil {
+			return false, err
+		}
+
+		if int32(len(gsList)) != flt.Spec.Replicas {
+			return false, nil
+		}
+
+		if err != nil {
+			return false, err
+		}
+
+		for _, gs := range gsList {
+			if !cond(gs) {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+}
+
+// CleanUp Delete all Agones resources in a given namespace
 func (f *Framework) CleanUp(ns string) error {
+	logrus.Info("Done. Cleaning up now.")
 	err := f.AgonesClient.StableV1alpha1().Fleets(ns).DeleteCollection(&metav1.DeleteOptions{}, metav1.ListOptions{})
 	if err != nil {
 		return err

test/e2e/main_test.go

@@ -33,8 +33,8 @@ func TestMain(m *testing.M) {
 	usr, _ := user.Current()
 	kubeconfig := flag.String("kubeconfig", filepath.Join(usr.HomeDir, "/.kube/config"),
 		"kube config path, e.g. $HOME/.kube/config")
-	gsimage := flag.String("gameserver-image", "gcr.io/agones-images/udp-server:0.3",
-		"gameserver image to use for those tests, gcr.io/agones-images/udp-server:0.3")
+	gsimage := flag.String("gameserver-image", "gcr.io/agones-images/udp-server:0.4",
+		"gameserver image to use for those tests, gcr.io/agones-images/udp-server:0.4")
 
 	flag.Parse()