Add acceptance test for crashed app instance stats

apps/crashing.go

@@ -1,19 +1,47 @@
 package apps
 
 import (
+	"encoding/json"
+	"fmt"
 	. "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers"
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-	. "github.com/onsi/gomega/gbytes"
-	. "github.com/onsi/gomega/gexec"
-
 	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
 	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
 	"github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name"
 	"github.com/cloudfoundry/cf-test-helpers/v2/cf"
 	"github.com/cloudfoundry/cf-test-helpers/v2/helpers"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gbytes"
+	. "github.com/onsi/gomega/gexec"
+	"time"
 )
 
+func hasOneInstanceInState(processPath, desiredState string) bool {
+	// Perform the CF curl command to get process stats
+	session := cf.Cf("curl", processPath).Wait()
+
+	// Parse the JSON response
+	instancesJson := struct {
+		Resources []struct {
+			Type  string `json:"type"`
+			State string `json:"state"`
+		} `json:"resources"`
+	}{}
+
+	// Read the session output and unmarshal the JSON data
+	bytes := session.Wait().Out.Contents()
+	err := json.Unmarshal(bytes, &instancesJson)
+	Expect(err).ToNot(HaveOccurred(), "Error unmarshalling process stats JSON")
+
+	// Check if any instance is in the desired state
+	for _, instance := range instancesJson.Resources {
+		if instance.State == desiredState {
+			return true
+		}
+	}
+	return false
+}
+
 var _ = AppsDescribe("Crashing", func() {
 	var appName string
 
@@ -45,6 +73,36 @@ var _ = AppsDescribe("Crashing", func() {
 		})
 	})
 
+	Describe("an app with three instances, two running and one crashing", func() {
+		It("keeps two instances running while another crashes", func() {
+			By("Pushing the app with three instances")
+			Expect(cf.Cf(
+				"push", appName,
+				"-b", "python_buildpack",
+				"-m", DEFAULT_MEMORY_LIMIT,
+				"-p", assets.NewAssets().PythonCrashApp,
+				"-i", "3", // Setting three instances
+			).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
+
+			By("Checking that the app is up and running")
+			Eventually(cf.Cf("app", appName)).Should(Say("running"))
+
+			By("Waiting until one instance crashes")
+			appGuid := app_helpers.GetAppGuid(appName)
+			processStatsPath := fmt.Sprintf("/v3/apps/%s/processes/web/stats", appGuid)
+
+			// Poll until at least one instance has crashed
+			Eventually(func() bool {
+				return hasOneInstanceInState(processStatsPath, "CRASHED")
+			}, 60*time.Second, 5*time.Second).Should(BeTrue(), "At least one instance should be in the CRASHED state")
+
+			By("Verifying at least one instance is still running")
+			foundRunning := hasOneInstanceInState(processStatsPath, "RUNNING")
+			Expect(foundRunning).To(BeTrue(), "At least one instance should still be in the RUNNING state")
+
+		})
+	})
+
 	Context("the app crashes", func() {
 		BeforeEach(func() {
 			Expect(cf.Cf(app_helpers.CatnipWithArgs(

assets/python-crash-app/Procfile

@@ -0,0 +1 @@
+web: python app.py

assets/python-crash-app/app.py

@@ -0,0 +1,29 @@
+import os
+import http.server
+import socketserver
+import sys
+
+class CustomHandler(http.server.SimpleHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+        message = f"Hello, you've reached the instance {os.getenv('CF_INSTANCE_INDEX', 'not defined')}!"
+        self.wfile.write(message.encode('utf-8'))
+
+def main():
+    port = int(os.getenv('PORT', 8080))
+    instance_index = int(os.getenv('CF_INSTANCE_INDEX', -1))
+
+    if instance_index > 1:
+        print(f"Instance {instance_index} is quitting!")
+        sys.exit(1)
+
+    handler = CustomHandler
+    httpd = socketserver.TCPServer(("", port), handler)
+
+    print(f"Serving on port {port} from instance {instance_index}")
+    httpd.serve_forever()
+
+if __name__ == "__main__":
+    main()

helpers/assets/assets.go

@@ -20,6 +20,7 @@ type Assets struct {
 	LoggregatorLoadGenerator   string
 	LoggregatorLoadGeneratorGo string
 	Python                     string
+	PythonCrashApp             string
 	Nginx                      string
 	Node                       string
 	CNBNode                    string
@@ -75,6 +76,7 @@ func NewAssets() Assets {
 		Php:                        "assets/php",
 		Proxy:                      "assets/proxy",
 		Python:                     "assets/python",
+		PythonCrashApp:             "assets/python-crash-app",
 		R:                          "assets/r",
 		RubySimple:                 "assets/ruby_simple",
 		SecurityGroupBuildpack:     "assets/security_group_buildpack.zip",