test: recovery after a broker restart

We know that recovering in-progress operations after a broker restart is
a non-ideal experience. These tests are to capture the current state of
the world, so that we can see an improvement when we fix the problems.

[#178142867](https://www.pivotaltracker.com/story/show/178142867)

integrationtest/fixtures/termination-recovery/fake-uuid-bind.tf

@@ -0,0 +1,3 @@
+resource "random_uuid" "random" {}
+
+output bind_output { value = random_uuid.random.result }

integrationtest/fixtures/termination-recovery/fake-uuid-provision.tf

@@ -0,0 +1,3 @@
+resource "random_uuid" "random" {}
+
+output provision_output { value = random_uuid.random.result }

integrationtest/fixtures/termination-recovery/fake-uuid-service.yml

@@ -0,0 +1,27 @@
+version: 1
+name: fake-uuid-service
+id: 083f2884-eb7b-11ee-96c7-174e35671015
+description: description
+display_name: Fake
+image_url: https://example.com/icon.jpg
+documentation_url: https://example.com
+support_url: https://example.com/support.html
+plans:
+- name: standard
+  id: 0d953850-eb7b-11ee-bb2c-8ba95d780d82
+  description: Standard plan
+  display_name: Standard
+provision:
+  template_refs:
+    main: fake-uuid-provision.tf
+  outputs:
+    - field_name: provision_output
+      type: string
+      details: provision output
+bind:
+  template_refs:
+    main: fake-uuid-bind.tf
+  outputs:
+    - field_name: bind_output
+      type: string
+      details: bind output

integrationtest/fixtures/termination-recovery/manifest.yml

@@ -0,0 +1,18 @@
+packversion: 1
+name: fake-brokerpak
+version: 0.1.0
+metadata:
+  author: noone@nowhere.com
+platforms:
+- os: linux
+  arch: amd64
+- os: darwin
+  arch: amd64
+terraform_binaries:
+- name: tofu
+  version: 1.6.0
+  source: https://github.com/opentofu/opentofu/archive/refs/tags/v1.6.0.zip
+- name: terraform-provider-random
+  version: 3.1.0
+service_definitions:
+- fake-uuid-service.yml

integrationtest/termination_recovery_test.go

@@ -0,0 +1,160 @@
+package integrationtest_test
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/cloudfoundry/cloud-service-broker/integrationtest/packer"
+	"github.com/cloudfoundry/cloud-service-broker/internal/testdrive"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gbytes"
+	"github.com/pborman/uuid"
+	"github.com/pivotal-cf/brokerapi/v10/domain"
+)
+
+var _ = Describe("Recovery From Broker Termination", func() {
+	const (
+		serviceOfferingGUID = "083f2884-eb7b-11ee-96c7-174e35671015"
+		servicePlanGUID     = "0d953850-eb7b-11ee-bb2c-8ba95d780d82"
+	)
+
+	var (
+		brokerpak string
+		broker    *testdrive.Broker
+		stdout    *Buffer
+		stderr    *Buffer
+	)
+
+	BeforeEach(func() {
+		brokerpak = must(packer.BuildBrokerpak(csb, fixtures("termination-recovery")))
+
+		stdout = NewBuffer()
+		stderr = NewBuffer()
+		broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr)))
+
+		DeferCleanup(func() {
+			Expect(broker.Stop()).To(Succeed())
+			cleanup(brokerpak)
+		})
+	})
+
+	It("cannot recover from a terminated create", func() {
+		By("starting to provision")
+		instanceGUID := uuid.New()
+		response := broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.New(), nil)
+		Expect(response.Error).NotTo(HaveOccurred())
+		Expect(response.StatusCode).To(Equal(http.StatusAccepted))
+
+		By("terminating and restarting the broker")
+		Expect(broker.Stop()).To(Succeed())
+		broker = must(testdrive.StartBroker(csb, brokerpak, database))
+
+		By("INCORRECTLY reporting that an operation is still in progress")
+		lastOperation, err := broker.LastOperation(instanceGUID)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(lastOperation.Description).To(Equal("provision in progress"))
+		Expect(lastOperation.State).To(BeEquivalentTo("in progress"))
+
+		// OSBAPI requires that HTTP 409 (Conflict) is returned
+		By("refusing to allow a duplicate instance")
+		response = broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.New(), nil)
+		Expect(response.Error).NotTo(HaveOccurred())
+		Expect(response.StatusCode).To(Equal(http.StatusConflict))
+
+		By("INCORRECTLY failing to allow the instance to be cleaned up")
+		response = broker.Client.Deprovision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.New())
+		Expect(response.Error).NotTo(HaveOccurred())
+		Expect(response.StatusCode).To(Equal(http.StatusInternalServerError))
+	})
+
+	It("can recover from a terminated update", func() {
+		By("successfully provisioning a service instance")
+		instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("starting to update")
+		response := broker.Client.Update(instance.GUID, serviceOfferingGUID, servicePlanGUID, uuid.New(), nil, domain.PreviousValues{}, nil)
+		Expect(response.Error).NotTo(HaveOccurred())
+		Expect(response.StatusCode).To(Equal(http.StatusAccepted))
+
+		By("terminating and restarting the broker")
+		Expect(broker.Stop()).To(Succeed())
+		broker = must(testdrive.StartBroker(csb, brokerpak, database))
+
+		By("INCORRECTLY reporting that an operation is still in progress")
+		lastOperation, err := broker.LastOperation(instance.GUID)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(lastOperation.Description).To(Equal("update in progress"))
+		Expect(lastOperation.State).To(BeEquivalentTo("in progress"))
+
+		By("allowing the operation to be restarted")
+		Expect(broker.UpdateService(instance)).To(Succeed())
+	})
+
+	It("can recover from a terminated delete", func() {
+		By("successfully provisioning a service instance")
+		instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("starting to delete")
+		response := broker.Client.Deprovision(instance.GUID, serviceOfferingGUID, servicePlanGUID, uuid.New())
+		Expect(response.Error).NotTo(HaveOccurred())
+		Expect(response.StatusCode).To(Equal(http.StatusAccepted))
+
+		By("terminating and restarting the broker")
+		Expect(broker.Stop()).To(Succeed())
+		broker = must(testdrive.StartBroker(csb, brokerpak, database))
+
+		By("INCORRECTLY reporting that an operation is still in progress")
+		lastOperation, err := broker.LastOperation(instance.GUID)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(lastOperation.Description).To(Equal("deprovision in progress"))
+		Expect(lastOperation.State).To(BeEquivalentTo("in progress"))
+
+		By("allowing the operation to be restarted")
+		Expect(broker.Deprovision(instance)).To(Succeed())
+	})
+
+	It("can recover from a terminated bind", func() {
+		By("successfully provisioning a service instance")
+		instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("starting to bind")
+		bindingGUID := uuid.New()
+		go broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID))
+
+		Eventually(stdout).Should(Say(fmt.Sprintf(`"cloud-service-broker.Binding".*"binding_id":"%s"`, bindingGUID)))
+
+		By("terminating and restarting the broker")
+		Expect(broker.Stop()).To(Succeed())
+		broker = must(testdrive.StartBroker(csb, brokerpak, database))
+
+		By("allowing the operation to be restarted")
+		_, err = broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID))
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("can recover from a terminated unbind", func() {
+		By("successfully provisioning a service instance and binding")
+		instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID)
+		Expect(err).NotTo(HaveOccurred())
+
+		bindingGUID := uuid.New()
+		_, err = broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID))
+		Expect(err).NotTo(HaveOccurred())
+
+		By("starting to unbind")
+		go broker.DeleteBinding(instance, bindingGUID)
+
+		Eventually(stdout).Should(Say(fmt.Sprintf(`"cloud-service-broker.Unbinding".*"binding_id":"%s"`, bindingGUID)))
+
+		By("terminating and restarting the broker")
+		Expect(broker.Stop()).To(Succeed())
+		broker = must(testdrive.StartBroker(csb, brokerpak, database))
+
+		By("allowing the operation to be restarted")
+		Expect(broker.DeleteBinding(instance, bindingGUID)).To(Succeed())
+	})
+})