From e8503accf8d22d6e73b44415236269561f902b4d Mon Sep 17 00:00:00 2001 From: Marek Brysa Date: Wed, 25 May 2022 08:26:58 +0200 Subject: [PATCH 1/5] storage/gs: do not set cluster_id if service_account is provided --- storage/gs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/gs.go b/storage/gs.go index a900e69605..14ae14466f 100644 --- a/storage/gs.go +++ b/storage/gs.go @@ -75,7 +75,7 @@ func createOrValidateClusterForGoogleStorage(ctx context.Context, m interface{}, if err != nil { return fmt.Errorf("cannot create mounting cluster: %w", err) } - return d.Set("cluster_id", cluster.ClusterID) + return nil } return fmt.Errorf("either cluster_id or service_account must be specified to mount GCS bucket") } From e40bdddd30a5f0fa8cfa192beb269771c7dcf9a6 Mon Sep 17 00:00:00 2001 From: Marek Brysa Date: Wed, 25 May 2022 09:46:37 +0200 Subject: [PATCH 2/5] set cluster_id, but retry --- storage/gs.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/storage/gs.go b/storage/gs.go index 14ae14466f..f07ce49b1e 100644 --- a/storage/gs.go +++ b/storage/gs.go @@ -63,6 +63,13 @@ func createOrValidateClusterForGoogleStorage(ctx context.Context, m interface{}, clustersAPI := clusters.NewClustersAPI(ctx, m) if clusterID != "" { clusterInfo, err := clustersAPI.Get(clusterID) + if common.IsMissing(err) { + cluster, err := GetOrCreateMountingClusterWithGcpServiceAccount(clustersAPI, serviceAccount) + if err != nil { + return fmt.Errorf("cannot re-create mounting cluster: %w", err) + } + return d.Set("cluster_id", cluster.ClusterID) + } if err != nil { return fmt.Errorf("cannot get mounting cluster: %w", err) } @@ -75,7 +82,7 @@ func createOrValidateClusterForGoogleStorage(ctx context.Context, m interface{}, if err != nil { return fmt.Errorf("cannot create mounting cluster: %w", err) } - return nil + return d.Set("cluster_id", cluster.ClusterID) } return fmt.Errorf("either cluster_id or service_account must be specified to mount GCS bucket") } From 19e865b5e25d9c8c2f3597ddbc333398e251895b Mon Sep 17 00:00:00 2001 From: Marek Brysa Date: Tue, 31 May 2022 08:18:59 +0200 Subject: [PATCH 3/5] fix test --- storage/gs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/gs_test.go b/storage/gs_test.go index 4299ea7b0d..38069126d7 100644 --- a/storage/gs_test.go +++ b/storage/gs_test.go @@ -20,7 +20,7 @@ func TestCreateOrValidateClusterForGoogleStorage_Failures(t *testing.T) { }, func(ctx context.Context, client *common.DatabricksClient) { d := ResourceMount().TestResourceData() err := createOrValidateClusterForGoogleStorage(ctx, client, d, "a", "") - assert.EqualError(t, err, "cannot get mounting cluster: nope") + assert.EqualError(t, err, "cannot re-create mounting cluster: nope") err = createOrValidateClusterForGoogleStorage(ctx, client, d, "", "b") assert.EqualError(t, err, "cannot create mounting cluster: nope") From c185c6f79ed4595de96442358d17523580fa6c9a Mon Sep 17 00:00:00 2001 From: Marek Brysa Date: Tue, 31 May 2022 08:19:13 +0200 Subject: [PATCH 4/5] add success test --- storage/gs_test.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/storage/gs_test.go b/storage/gs_test.go index 38069126d7..3aaf71b31d 100644 --- a/storage/gs_test.go +++ b/storage/gs_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/databrickslabs/terraform-provider-databricks/clusters" "github.com/databrickslabs/terraform-provider-databricks/common" "github.com/databrickslabs/terraform-provider-databricks/qa" "github.com/stretchr/testify/assert" @@ -26,3 +27,70 @@ func TestCreateOrValidateClusterForGoogleStorage_Failures(t *testing.T) { assert.EqualError(t, err, "cannot create mounting cluster: nope") }) } + +func TestCreateOrValidateClusterForGoogleStorage_WorksOnDeletedCluster(t *testing.T) { + qa.HTTPFixturesApply(t, []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/clusters/get?cluster_id=removed-cluster", + Status: 404, + Response: common.NotFound("cluster deleted"), + }, + { + Method: "GET", + Resource: "/api/2.0/clusters/list", + Response: clusters.ClusterList{ + Clusters: []clusters.ClusterInfo{}, + }, + }, + { + ReuseRequest: true, + Method: "GET", + Resource: "/api/2.0/clusters/spark-versions", + }, + { + ReuseRequest: true, + Method: "GET", + Resource: "/api/2.0/clusters/list-node-types", + }, + { + Method: "POST", + Resource: "/api/2.0/clusters/create", + ExpectedRequest: clusters.Cluster{ + CustomTags: map[string]string{ + "ResourceClass": "SingleNode", + }, + ClusterName: "terraform-mount-gcs-03a56ec1d1576b505aabf088337cbf36", + GcpAttributes: &clusters.GcpAttributes{ + GoogleServiceAccount: "service-account", + }, + SparkVersion: "7.3.x-scala2.12", + NumWorkers: 0, + NodeTypeID: "i3.xlarge", + AutoterminationMinutes: 10, + SparkConf: map[string]string{ + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*]", + "spark.scheduler.mode": "FIFO", + }, + }, + Response: clusters.ClusterID{ + ClusterID: "new-cluster", + }, + }, + { + Method: "GET", + Resource: "/api/2.0/clusters/get?cluster_id=new-cluster", + Response: clusters.ClusterInfo{ + ClusterID: "new-cluster", + State: "RUNNING", + StateMessage: "created", + }, + }, + }, func(ctx context.Context, client *common.DatabricksClient) { + d := ResourceMount().TestResourceData() + err := createOrValidateClusterForGoogleStorage(ctx, client, d, "removed-cluster", "service-account") + assert.NoError(t, err) + assert.Equal(t, "new-cluster", d.Get("cluster_id")) + }) +} From 638b9f530039c8ed65ead214350ff009289e195c Mon Sep 17 00:00:00 2001 From: Marek Brysa Date: Tue, 31 May 2022 15:32:27 +0200 Subject: [PATCH 5/5] add test --- storage/gs_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/storage/gs_test.go b/storage/gs_test.go index 3aaf71b31d..7b782f369b 100644 --- a/storage/gs_test.go +++ b/storage/gs_test.go @@ -94,3 +94,22 @@ func TestCreateOrValidateClusterForGoogleStorage_WorksOnDeletedCluster(t *testin assert.Equal(t, "new-cluster", d.Get("cluster_id")) }) } + +func TestCreateOrValidateClusterForGoogleStorage_FailsOnErrorGettingCluster(t *testing.T) { + qa.HTTPFixturesApply(t, []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/clusters/get?cluster_id=my-cluster", + Status: 500, + Response: common.APIError{ + ErrorCode: "SERVER_ERROR", + StatusCode: 500, + Message: "Server error", + }, + }, + }, func(ctx context.Context, client *common.DatabricksClient) { + d := ResourceMount().TestResourceData() + err := createOrValidateClusterForGoogleStorage(ctx, client, d, "my-cluster", "service-account") + assert.EqualError(t, err, "cannot get mounting cluster: Server error") + }) +}