From 98181881087640147055e1b1ea5fabe4f3d3146e Mon Sep 17 00:00:00 2001 From: Felisia Martini Date: Wed, 21 Aug 2024 12:55:11 +0100 Subject: [PATCH] feat: Experimental - Possibility to import state into CSB (#1074) * spike: alternative to subsume [#187405764](https://www.pivotaltracker.com/story/show/187405764) * test: use new uuid package * chore: improve uuid validation * test: the format function is not needed --------- Co-authored-by: George Blue Co-authored-by: Andrea Zucchini --- brokerapi/broker/broker.go | 4 +- brokerapi/broker/provision.go | 1 + cmd/serve.go | 55 +++++++++++- .../import-state-data/terraform.tfstate | 36 ++++++++ .../import-state/fake-uuid-provision.tf | 8 ++ .../import-state/fake-uuid-service.yml | 25 ++++++ .../fixtures/import-state/manifest.yml | 18 ++++ .../fixtures/import-state/versions.tf | 7 ++ integrationtest/import_state_test.go | 85 +++++++++++++++++++ internal/testdrive/broker.go | 4 +- internal/testdrive/broker_start.go | 4 +- pkg/providers/tf/provider.go | 7 +- 12 files changed, 245 insertions(+), 9 deletions(-) create mode 100644 integrationtest/fixtures/import-state-data/terraform.tfstate create mode 100644 integrationtest/fixtures/import-state/fake-uuid-provision.tf create mode 100644 integrationtest/fixtures/import-state/fake-uuid-service.yml create mode 100644 integrationtest/fixtures/import-state/manifest.yml create mode 100644 integrationtest/fixtures/import-state/versions.tf create mode 100644 integrationtest/import_state_test.go diff --git a/brokerapi/broker/broker.go b/brokerapi/broker/broker.go index 1004e9648..2799ab0a8 100755 --- a/brokerapi/broker/broker.go +++ b/brokerapi/broker/broker.go @@ -86,7 +86,9 @@ func validateNoPlanParametersOverrides(params map[string]any, plan *broker.Servi } func validateDefinedParams(params map[string]any, validUserInputFields []broker.BrokerVariable, validImportFields []broker.ImportVariable) error { - validParams := make(map[string]struct{}) + validParams := map[string]struct{}{ + "vacant": {}, + } for _, field := range validUserInputFields { validParams[field.FieldName] = struct{}{} } diff --git a/brokerapi/broker/provision.go b/brokerapi/broker/provision.go index b023af7c1..be3104966 100644 --- a/brokerapi/broker/provision.go +++ b/brokerapi/broker/provision.go @@ -99,6 +99,7 @@ func (broker *ServiceBroker) Provision(ctx context.Context, instanceID string, d } // save provision request details + delete(parsedDetails.RequestParams, "vacant") if err := broker.store.StoreProvisionRequestDetails(instanceID, parsedDetails.RequestParams); err != nil { return domain.ProvisionedServiceSpec{}, fmt.Errorf("error saving provision request details to database: %s. Services relying on async provisioning will not be able to complete provisioning", err) } diff --git a/cmd/serve.go b/cmd/serve.go index 9d0a376d8..feb61bf7b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -17,7 +17,9 @@ package cmd import ( "context" "database/sql" + "encoding/json" "fmt" + "io" "log/slog" "net/http" "strings" @@ -31,10 +33,13 @@ import ( "github.com/cloudfoundry/cloud-service-broker/v2/internal/storage" pakBroker "github.com/cloudfoundry/cloud-service-broker/v2/pkg/broker" "github.com/cloudfoundry/cloud-service-broker/v2/pkg/brokerpak" + "github.com/cloudfoundry/cloud-service-broker/v2/pkg/providers/tf/workspace" "github.com/cloudfoundry/cloud-service-broker/v2/pkg/server" "github.com/cloudfoundry/cloud-service-broker/v2/pkg/toggles" "github.com/cloudfoundry/cloud-service-broker/v2/utils" + "github.com/google/uuid" "github.com/pivotal-cf/brokerapi/v11" + "github.com/pivotal-cf/brokerapi/v11/auth" "github.com/pivotal-cf/brokerapi/v11/domain" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -124,7 +129,7 @@ func serve() { if err != nil { logger.Error("failed to get database connection", err) } - startServer(cfg.Registry, sqldb, brokerAPI) + startServer(cfg.Registry, sqldb, brokerAPI, storage.New(db, encryptor), credentials) } func serveDocs() { @@ -135,7 +140,7 @@ func serveDocs() { logger.Error("loading brokerpaks", err) } - startServer(registry, nil, nil) + startServer(registry, nil, nil, nil, brokerapi.BrokerCredentials{}) } func setupDBEncryption(db *gorm.DB, logger lager.Logger) storage.Encryptor { @@ -179,7 +184,7 @@ func setupDBEncryption(db *gorm.DB, logger lager.Logger) storage.Encryptor { return config.Encryptor } -func startServer(registry pakBroker.BrokerRegistry, db *sql.DB, brokerapi http.Handler) { +func startServer(registry pakBroker.BrokerRegistry, db *sql.DB, brokerapi http.Handler, store *storage.Storage, credentials brokerapi.BrokerCredentials) { logger := utils.NewLogger("cloud-service-broker") docsHandler := server.DocsHandler(registry) @@ -189,6 +194,7 @@ func startServer(registry pakBroker.BrokerRegistry, db *sql.DB, brokerapi http.H router.HandleFunc("/examples", server.NewExampleHandler(registry)) server.AddHealthHandler(router, db) router.HandleFunc("/info", infohandler.NewDefault()) + router.Handle("/import_state/{guid}", auth.NewWrapper(credentials.Username, credentials.Password).Wrap(importStateHandler(store))) router.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) { switch { @@ -215,3 +221,46 @@ func labelName(label string) string { return label } } + +func importStateHandler(store *storage.Storage) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + guid := r.PathValue("guid") + if guid == "" { + http.Error(w, "GUID is required", http.StatusBadRequest) + return + } + if err := uuid.Validate(guid); err != nil { + http.Error(w, "not a valid GUID", http.StatusBadRequest) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, fmt.Sprintf("failed to read body: %s", err), http.StatusBadRequest) + return + } + var b any + if err := json.Unmarshal(body, &b); err != nil { + http.Error(w, fmt.Sprintf("problem parsing body as JSON: %s", err), http.StatusBadRequest) + return + } + + tfID := fmt.Sprintf("tf:%s:", guid) + deployment, err := store.GetTerraformDeployment(tfID) + if err != nil { + http.Error(w, fmt.Sprintf("could not find TF ID: %s", tfID), http.StatusNotFound) + return + } + + tfw, ok := deployment.Workspace.(*workspace.TerraformWorkspace) + if !ok { + http.Error(w, "failed cast to *workspace.TerraformWorkspace", http.StatusInternalServerError) + return + } + + tfw.State = body + if err := store.StoreTerraformDeployment(deployment); err != nil { + http.Error(w, fmt.Sprintf("failed to store deployment: %s", err), http.StatusInternalServerError) + return + } + }) +} diff --git a/integrationtest/fixtures/import-state-data/terraform.tfstate b/integrationtest/fixtures/import-state-data/terraform.tfstate new file mode 100644 index 000000000..721d35c2a --- /dev/null +++ b/integrationtest/fixtures/import-state-data/terraform.tfstate @@ -0,0 +1,36 @@ +{ + "version": 4, + "terraform_version": "1.6.2", + "serial": 2, + "lineage": "f4f27494-7de8-a572-4e5b-d2530202be8f", + "outputs": { + "provision_output": { + "value": "831e0be4-7ff3-26ac-751c-80951adb3fe7", + "type": "string" + }, + "status": { + "value": "created random GUID: 831e0be4-7ff3-26ac-751c-80951adb3fe7", + "type": "string" + } + }, + "resources": [ + { + "mode": "managed", + "type": "random_uuid", + "name": "random", + "provider": "provider[\"registry.terraform.io/hashicorp/random\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "831e0be4-7ff3-26ac-751c-80951adb3fe7", + "keepers": null, + "result": "831e0be4-7ff3-26ac-751c-80951adb3fe7" + }, + "sensitive_attributes": [] + } + ] + } + ], + "check_results": null +} diff --git a/integrationtest/fixtures/import-state/fake-uuid-provision.tf b/integrationtest/fixtures/import-state/fake-uuid-provision.tf new file mode 100644 index 000000000..3b07c8df8 --- /dev/null +++ b/integrationtest/fixtures/import-state/fake-uuid-provision.tf @@ -0,0 +1,8 @@ +resource "random_uuid" "random" { +} + +output provision_output { value = random_uuid.random.result } + +output "status" { + value = format("created random GUID: %s", random_uuid.random.result) +} \ No newline at end of file diff --git a/integrationtest/fixtures/import-state/fake-uuid-service.yml b/integrationtest/fixtures/import-state/fake-uuid-service.yml new file mode 100644 index 000000000..0503cd97c --- /dev/null +++ b/integrationtest/fixtures/import-state/fake-uuid-service.yml @@ -0,0 +1,25 @@ +version: 1 +name: fake-uuid-service +id: 5b4f6244-f7ee-11ee-b5b3-3389c8712346 +description: description +display_name: Fake +image_url: https://example.com/icon.jpg +documentation_url: https://example.com +support_url: https://example.com/support.html +plans: +- name: default + id: 5b50951a-f7ee-11ee-b564-6b989de50807 + description: default plan + display_name: default +provision: + template_refs: + main: fake-uuid-provision.tf + versions: versions.tf + outputs: + - field_name: provision_output + type: string + details: provision output + user_inputs: + - field_name: value + type: string + details: input diff --git a/integrationtest/fixtures/import-state/manifest.yml b/integrationtest/fixtures/import-state/manifest.yml new file mode 100644 index 000000000..306bb39ab --- /dev/null +++ b/integrationtest/fixtures/import-state/manifest.yml @@ -0,0 +1,18 @@ +packversion: 1 +name: fake-brokerpak +version: 0.1.0 +metadata: + author: noone@nowhere.com +platforms: +- os: linux + arch: amd64 +- os: darwin + arch: amd64 +terraform_binaries: +- name: tofu + version: 1.6.0 + source: https://github.com/opentofu/opentofu/archive/refs/tags/v1.6.0.zip +- name: terraform-provider-random + version: 3.1.0 +service_definitions: +- fake-uuid-service.yml diff --git a/integrationtest/fixtures/import-state/versions.tf b/integrationtest/fixtures/import-state/versions.tf new file mode 100644 index 000000000..09be45e87 --- /dev/null +++ b/integrationtest/fixtures/import-state/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + random = { + source = "registry.terraform.io/hashicorp/random" + } + } +} diff --git a/integrationtest/import_state_test.go b/integrationtest/import_state_test.go new file mode 100644 index 000000000..a02286d48 --- /dev/null +++ b/integrationtest/import_state_test.go @@ -0,0 +1,85 @@ +package integrationtest_test + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "net/http" + + "github.com/cloudfoundry/cloud-service-broker/v2/dbservice/models" + "github.com/cloudfoundry/cloud-service-broker/v2/integrationtest/packer" + "github.com/cloudfoundry/cloud-service-broker/v2/internal/testdrive" + "github.com/cloudfoundry/cloud-service-broker/v2/pkg/providers/tf/workspace" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/brokerapi/v11/domain" +) + +//go:embed "fixtures/import-state-data/terraform.tfstate" +var stateToImport []byte + +var _ = Describe("Import State", func() { + var ( + brokerpak string + broker *testdrive.Broker + ) + + BeforeEach(func() { + brokerpak = must(packer.BuildBrokerpak(csb, fixtures("import-state"))) + broker = must(testdrive.StartBroker(csb, brokerpak, database)) + + DeferCleanup(func() { + Expect(broker.Stop()).To(Succeed()) + cleanup(brokerpak) + }) + }) + + It("can create a vacant service instance and import a terraform state", func() { + const ( + serviceOfferingGUID = "5b4f6244-f7ee-11ee-b5b3-3389c8712346" + servicePlanGUID = "5b50951a-f7ee-11ee-b564-6b989de50807" + importedValue = "831e0be4-7ff3-26ac-751c-80951adb3fe7" // matches what's in the test fixture + ) + + By("creating a 'vacant' service instance") + instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID, testdrive.WithProvisionParams(`{"vacant":true}`)) + Expect(err).NotTo(HaveOccurred()) + + By("checking that the state is empty") + var d models.TerraformDeployment + terraformDeploymentID := fmt.Sprintf("tf:%s:", instance.GUID) + Expect(dbConn.Where("id = ?", terraformDeploymentID).First(&d).Error).To(Succeed()) + var w workspace.TerraformWorkspace + Expect(json.Unmarshal(d.Workspace, &w)).To(Succeed()) + Expect(w.State).To(MatchJSON(`{"version":4}`)) + + By("checking that the `vacant` parameter was not stored") + var i models.ProvisionRequestDetails + Expect(dbConn.Where("service_instance_id = ?", instance.GUID).First(&i).Error).To(Succeed()) + Expect(i.RequestDetails).To(MatchJSON(`{}`)) + + By("importing state into the vacant service instance") + req := must(http.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:%d/import_state/%s", broker.Port, instance.GUID), bytes.NewReader(stateToImport))) + req.SetBasicAuth(broker.Username, broker.Password) + importResponse := must(http.DefaultClient.Do(req)) + Expect(importResponse).To(HaveHTTPStatus(http.StatusOK)) + + By("checking that the state was imported into the database") + Expect(dbConn.Where("id = ?", terraformDeploymentID).First(&d).Error).To(Succeed()) + Expect(json.Unmarshal(d.Workspace, &w)).To(Succeed()) + Expect(w.State).To(MatchJSON(stateToImport)) + + By("performing a no-op update to trigger an Apply") + updateResponse := broker.Client.Update(instance.GUID, instance.ServiceOfferingGUID, servicePlanGUID, uuid.NewString(), nil, domain.PreviousValues{}, nil) + Expect(updateResponse.Error).NotTo(HaveOccurred()) + Expect(updateResponse.StatusCode).To(Equal(http.StatusAccepted)) + state, err := broker.LastOperationFinalValue(instance.GUID) + Expect(err).NotTo(HaveOccurred()) + Expect(state.State).To(BeEquivalentTo("succeeded")) + + By("checking that data from the imported state made it into the Last Operation output") + Expect(state.Description).To(Equal(fmt.Sprintf("update succeeded: created random GUID: %s", importedValue))) + }) +}) diff --git a/internal/testdrive/broker.go b/internal/testdrive/broker.go index 953c0ead1..416d70deb 100644 --- a/internal/testdrive/broker.go +++ b/internal/testdrive/broker.go @@ -12,8 +12,8 @@ type Broker struct { Port int Client *client.Client runner *runner - username string - password string + Username string + Password string Stdout *bytes.Buffer Stderr *bytes.Buffer } diff --git a/internal/testdrive/broker_start.go b/internal/testdrive/broker_start.go index 3dfb44efb..edc506cd1 100644 --- a/internal/testdrive/broker_start.go +++ b/internal/testdrive/broker_start.go @@ -74,8 +74,8 @@ func StartBroker(csbPath, bpk, db string, opts ...StartBrokerOption) (*Broker, e Database: db, Port: port, Client: clnt, - username: username, - password: password, + Username: username, + Password: password, runner: newCommand(cmd), Stdout: &stdout, Stderr: &stderr, diff --git a/pkg/providers/tf/provider.go b/pkg/providers/tf/provider.go index 3fa9e6f03..6c9828995 100644 --- a/pkg/providers/tf/provider.go +++ b/pkg/providers/tf/provider.go @@ -90,7 +90,12 @@ func (provider *TerraformProvider) create(ctx context.Context, vars *varcontext. } go func() { - err := provider.DefaultInvoker().Apply(ctx, newWorkspace) + var err error + if vars.HasKey("vacant") && vars.GetBool("vacant") { + newWorkspace.State = []byte(`{"version":4}`) // Minimum state required for anything to work + } else { + err = provider.DefaultInvoker().Apply(ctx, newWorkspace) + } _ = provider.MarkOperationFinished(&deployment, err) }()