Added very simple stress test which scales fleets up/down repeatedly …

…and basic stress test harness.

The same test is used during regular e2e tests, except it runs just a few iterations on smaller fleets.

To run stress test simply invoke `make stress-test-e2e` optionally passing `STRESS_TEST_LEVEL`, which controls the fleet sizes to be used (1..100, defaults to 20). Depending on stress test level, you may need a cluster with lots of capacity.

By convention 'make stress-test-e2e' runs all test cases whose names include 'StressTest' and ignores everything else.

build/Makefile

@@ -48,6 +48,10 @@ GCP_BUCKET_CHARTS ?= agones-chart
 MINIKUBE_PROFILE ?= agones
 GO_BUILD_TAGS ?= none
 
+# Specify stress test level 1..100
+# STRESS_TEST_LEVEL=n requires capacity between 50*n up to 100*n simple-udp Game Servers.
+STRESS_TEST_LEVEL=20
+
 # kind cluster name to use
 KIND_PROFILE ?= agones
 KIND_CONTAINER_NAME=kind-$(KIND_PROFILE)-control-plane
@@ -217,6 +221,16 @@ test-e2e: $(ensure-build-image)
 		--gameserver-image=$(GS_TEST_IMAGE) \
 		--pullsecret=$(IMAGE_PULL_SECRET)
 
+# Runs end-to-end stress tests on the current configured cluster
+# For minikube user the minikube-stress-test-e2e targets
+stress-test-e2e: $(ensure-build-image)
+	$(GO_TEST) $(agones_package)/test/e2e $(ARGS) $(GO_E2E_TEST_ARGS) \
+		-timeout 1h \
+		-run '.*StressTest.*' \
+		--gameserver-image=$(GS_TEST_IMAGE) \
+		--pullsecret=$(IMAGE_PULL_SECRET) \
+		--stress $(STRESS_TEST_LEVEL)
+
 # Run test on install yaml - make sure there is no change
 # mostly this is for CI
 test-install-yaml:
@@ -610,6 +624,10 @@ minikube-transfer-image:
 minikube-test-e2e: DOCKER_RUN_ARGS=--network=host -v $(minikube_cert_mount)
 minikube-test-e2e: minikube-agones-profile test-e2e
 
+# Runs stress tests against our minikube
+minikube-stress-test-e2e: DOCKER_RUN_ARGS=--network=host -v $(minikube_cert_mount)
+minikube-stress-test-e2e: minikube-agones-profile stress-test-e2e
+
 # prometheus on minkube 
 # we have to disable PVC as it's not supported on minkube.
 minikube-setup-prometheus:

test/e2e/fleet_test.go

@@ -16,6 +16,7 @@ package e2e
 
 import (
 	"fmt"
+	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -640,6 +641,116 @@ func TestCreateFleetAndUpdateScaleSubresource(t *testing.T) {
 	framework.WaitForFleetCondition(t, flt, e2e.FleetReadyCount(initialReplicas))
 }
 
+// TestScaleUpAndDownInParallelStressTest creates N fleets, half of which start with replicas=0
+// and the other half with 0 and scales them up/down 3 times in parallel expecting it to reach
+// the desired number of ready replicas each time.
+// This test is also used as a stress test with 'make stress-test-e2e', in which case it creates
+// many more fleets of bigger sizes and runs many more repetitions.
+func TestScaleUpAndDownInParallelStressTest(t *testing.T) {
+	t.Parallel()
+
+	alpha1 := framework.AgonesClient.StableV1alpha1()
+	fleetCount := 2
+	fleetSize := int32(10)
+	repeatCount := 3
+
+	logrus.WithField("fleetCount", fleetCount).WithField("fleetSize", fleetSize).WithField("repeatCount", repeatCount).Info("starting scale up/down test")
+
+	if framework.StressTestLevel > 0 {
+		fleetSize = 10 * int32(framework.StressTestLevel)
+		repeatCount = 20
+		fleetCount = 10
+	}
+
+	var fleets []*v1alpha1.Fleet
+
+	var (
+		mu               sync.Mutex
+		scaleUpSamples   []time.Duration
+		scaleDownSamples []time.Duration
+	)
+
+	addSample := func(samplesSlice *[]time.Duration, sample time.Duration) {
+		mu.Lock()
+		*samplesSlice = append(*samplesSlice, sample)
+		mu.Unlock()
+	}
+
+	for fleetNumber := 0; fleetNumber < fleetCount; fleetNumber++ {
+		flt := defaultFleet()
+		flt.ObjectMeta.GenerateName = fmt.Sprintf("scale-fleet-%v-", fleetNumber)
+		if fleetNumber%2 == 0 {
+			// even-numbered fleets starts at fleetSize and are scaled down to zero and back.
+			flt.Spec.Replicas = fleetSize
+		} else {
+			// odd-numbered fleets starts at zero and are scaled up to fleetSize and back.
+			flt.Spec.Replicas = 0
+		}
+
+		flt, err := alpha1.Fleets(defaultNs).Create(flt)
+		if assert.Nil(t, err) {
+			defer alpha1.Fleets(defaultNs).Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck
+		}
+		fleets = append(fleets, flt)
+	}
+
+	// wait for initial fleet conditions.
+	for fleetNumber, flt := range fleets {
+		if fleetNumber%2 == 0 {
+			framework.WaitForFleetCondition(t, flt, e2e.FleetReadyCount(fleetSize))
+		} else {
+			framework.WaitForFleetCondition(t, flt, e2e.FleetReadyCount(0))
+		}
+	}
+
+	var wg sync.WaitGroup
+
+	for fleetNumber, flt := range fleets {
+		wg.Add(1)
+		go func(fleetNumber int, flt *v1alpha1.Fleet) {
+			defer wg.Done()
+			defer func() {
+				if err := recover(); err != nil {
+					t.Errorf("recovered panic: %v", err)
+				}
+			}()
+
+			if fleetNumber%2 == 0 {
+				addSample(&scaleDownSamples, scaleAndWait(t, flt, 0))
+			}
+			for i := 0; i < repeatCount; i++ {
+				addSample(&scaleUpSamples, scaleAndWait(t, flt, fleetSize))
+				addSample(&scaleDownSamples, scaleAndWait(t, flt, 0))
+			}
+		}(fleetNumber, flt)
+	}
+
+	wg.Wait()
+
+	sort.Slice(scaleUpSamples, func(i, j int) bool {
+		return scaleUpSamples[i] < scaleUpSamples[j]
+	})
+	sort.Slice(scaleDownSamples, func(i, j int) bool {
+		return scaleDownSamples[i] < scaleDownSamples[j]
+	})
+
+	for i, s := range scaleUpSamples {
+		t.Logf("scaleUp[%v] = %v", i, s)
+	}
+	for i, s := range scaleDownSamples {
+		t.Logf("scaleDown[%v] = %v", i, s)
+	}
+
+	// TODO - can we publish those numbers somewhere on a regular basis?
+}
+
+func scaleAndWait(t *testing.T, flt *v1alpha1.Fleet, fleetSize int32) time.Duration {
+	t0 := time.Now()
+	scaleFleetSubresource(t, flt, fleetSize)
+	framework.WaitForFleetCondition(t, flt, e2e.FleetReadyCount(fleetSize))
+	return time.Since(t0)
+}
+
 // scaleFleetPatch creates a patch to apply to a Fleet.
 // Easier for testing, as it removes object generational issues.
 func scaleFleetPatch(t *testing.T, f *v1alpha1.Fleet, scale int32) *v1alpha1.Fleet {
@@ -654,6 +765,7 @@ func scaleFleetPatch(t *testing.T, f *v1alpha1.Fleet, scale int32) *v1alpha1.Fle
 // scaleFleetSubresource uses scale subresource to change Replicas size of the Fleet.
 // Returns the same f as in parameter, just to keep signature in sync with scaleFleetPatch
 func scaleFleetSubresource(t *testing.T, f *v1alpha1.Fleet, scale int32) *v1alpha1.Fleet {
+	logrus.WithField("scale", scale).WithField("fleet", f.Name).Info("scaling fleet using subresource")
 	alpha1 := framework.AgonesClient.StableV1alpha1()
 	// GetScale returns current Scale object with resourceVersion which is opaque object
 	// and it will be used to create new Scale object

test/e2e/framework/framework.go

@@ -48,10 +48,11 @@ type Framework struct {
 	AgonesClient    versioned.Interface
 	GameServerImage string
 	PullSecret      string
+	StressTestLevel int
 }
 
 // New setups a testing framework using a kubeconfig path and the game server image to use for testing.
-func New(kubeconfig, gsimage string, pullSecret string) (*Framework, error) {
+func New(kubeconfig, gsimage string, pullSecret string, stressTestLevel int) (*Framework, error) {
 	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
 		return nil, errors.Wrap(err, "build config from flags failed")
@@ -72,6 +73,7 @@ func New(kubeconfig, gsimage string, pullSecret string) (*Framework, error) {
 		AgonesClient:    agonesClient,
 		GameServerImage: gsimage,
 		PullSecret:      pullSecret,
+		StressTestLevel: stressTestLevel,
 	}, nil
 }
 
@@ -137,7 +139,7 @@ func (f *Framework) WaitForFleetCondition(t *testing.T, flt *v1alpha1.Fleet, con
 	})
 	if err != nil {
 		logrus.WithField("fleet", flt.Name).WithError(err).Info("error waiting for fleet condition")
-		t.Fatal("error waiting for fleet condition")
+		t.Fatalf("error waiting for fleet condition on fleet %v", flt.Name)
 	}
 }
 

test/e2e/main_test.go

@@ -29,6 +29,8 @@ const defaultNs = "default"
 
 var framework *e2eframework.Framework
 
+var stressTestLevel int
+
 func TestMain(m *testing.M) {
 	usr, _ := user.Current()
 	kubeconfig := flag.String("kubeconfig", filepath.Join(usr.HomeDir, "/.kube/config"),
@@ -37,6 +39,7 @@ func TestMain(m *testing.M) {
 		"gameserver image to use for those tests, gcr.io/agones-images/udp-server:0.6")
 	pullSecret := flag.String("pullsecret", "",
 		"optional secret to be used for pulling the gameserver and/or Agones SDK sidecar images")
+	stressTestLevel := flag.Int("stress", 0, "enable stress test at given level 0-100")
 
 	flag.Parse()
 
@@ -45,7 +48,7 @@ func TestMain(m *testing.M) {
 		exitCode int
 	)
 
-	if framework, err = e2eframework.New(*kubeconfig, *gsimage, *pullSecret); err != nil {
+	if framework, err = e2eframework.New(*kubeconfig, *gsimage, *pullSecret, *stressTestLevel); err != nil {
 		log.Printf("failed to setup framework: %v\n", err)
 		os.Exit(1)
 	}