Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(Cloud Databases): Create and Delete Logical Replication Slots for databases-for-postgresql #4116

Merged
merged 19 commits into from
Dec 1, 2022
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
7f636ce
added logical replication slot to schema and create function
omaraibrahim Oct 3, 2022
dc7ed24
added logical replication creation and deletion. it works :D
omaraibrahim Oct 5, 2022
349dccb
finished writing logical replication code, testing, and documenting
omaraibrahim Oct 18, 2022
4f61e10
updated secrets.baseline
omaraibrahim Oct 18, 2022
7a2fac0
reformated code so that we attempt updating the user password first. …
omaraibrahim Oct 27, 2022
44dd3dc
update users first and then create them. prevent use of logical repli…
omaraibrahim Oct 28, 2022
6257092
Merge branch 'master' into logical_replication
omaraibrahim Nov 8, 2022
23c3ce8
refactored logical replication update code according to review
omaraibrahim Nov 9, 2022
2b78695
updated error messages
omaraibrahim Nov 9, 2022
700ae1c
Merge branch 'master' into logical_replication
omaraibrahim Nov 11, 2022
e299e4c
fixed bug. changed to getok from getokexists
omaraibrahim Nov 11, 2022
84cf939
Merge branch 'IBM-Cloud:master' into logical_replication
omaraibrahim Nov 17, 2022
06aafc4
implemented logical replication on create
omaraibrahim Nov 17, 2022
1a7625d
modified tests
omaraibrahim Nov 17, 2022
866c817
Update database.html.markdown
omaraibrahim Nov 17, 2022
7a099a4
Merge branch 'master' into logical_replication
omaraibrahim Nov 18, 2022
a8bcaf1
removed accidental duplication
omaraibrahim Nov 23, 2022
99334ca
removed unnessary code block
omaraibrahim Nov 28, 2022
8a74dbb
fixed description typo
omaraibrahim Dec 1, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 28 additions & 12 deletions .secrets.baseline
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"files": "go.sum|^.secrets.baseline$|metadata",
"lines": null
},
"generated_at": "2022-09-15T09:19:18Z",
"generated_at": "2022-10-18T17:40:55Z",
"plugins_used": [
{
"name": "AWSKeyDetector"
Expand Down Expand Up @@ -650,7 +650,7 @@
"hashed_secret": "813274ccae5b6b509379ab56982d862f7b5969b6",
"is_secret": false,
"is_verified": false,
"line_number": 761,
"line_number": 775,
"type": "Base64 High Entropy String",
"verified_result": null
}
Expand Down Expand Up @@ -730,15 +730,15 @@
"hashed_secret": "c8b6f5ef11b9223ac35a5663975a466ebe7ebba9",
"is_secret": false,
"is_verified": false,
"line_number": 1452,
"line_number": 1478,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "8abf4899c01104241510ba87685ad4de76b0c437",
"is_secret": false,
"is_verified": false,
"line_number": 1458,
"line_number": 1484,
"type": "Secret Keyword",
"verified_result": null
}
Expand Down Expand Up @@ -1630,15 +1630,15 @@
"hashed_secret": "3046d9f6cfaaeea6eed9bb7a4ab010fe49b0cfd4",
"is_secret": false,
"is_verified": false,
"line_number": 1565,
"line_number": 1568,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "2c7d1e61c036dc18b2e9b3e6392c8e59c8437f23",
"is_secret": false,
"is_verified": false,
"line_number": 1895,
"line_number": 1914,
"type": "Secret Keyword",
"verified_result": null
}
Expand All @@ -1648,31 +1648,31 @@
"hashed_secret": "deab23f996709b4e3d14e5499d1cc2de677bfaa8",
"is_secret": false,
"is_verified": false,
"line_number": 1437,
"line_number": 1482,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "20a25bac21219ffff1904bde871ded4027eca2f8",
"is_secret": false,
"is_verified": false,
"line_number": 2035,
"line_number": 2084,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "b732fb611fd46a38e8667f9972e0cde777fbe37f",
"is_secret": false,
"is_verified": false,
"line_number": 2054,
"line_number": 2103,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
"is_secret": false,
"is_verified": false,
"line_number": 2326,
"line_number": 2463,
"type": "Secret Keyword",
"verified_result": null
}
Expand Down Expand Up @@ -1756,11 +1756,19 @@
}
],
"ibm/service/database/resource_ibm_database_postgresql_test.go": [
{
"hashed_secret": "e407cbe1c64cadb886be6f42907e2dd1c06ca080",
"is_secret": false,
"is_verified": false,
"line_number": 574,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "10c28f9cf0668595d45c1090a7b4a2ae98edfa58",
"is_secret": false,
"is_verified": false,
"line_number": 917,
"line_number": 934,
"type": "Secret Keyword",
"verified_result": null
}
Expand Down Expand Up @@ -3252,11 +3260,19 @@
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "e407cbe1c64cadb886be6f42907e2dd1c06ca080",
"is_secret": false,
"is_verified": false,
"line_number": 498,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "91199272d5d6a574a51722ca6f3d1148edb1a0e7",
"is_secret": false,
"is_verified": false,
"line_number": 488,
"line_number": 522,
"type": "Secret Keyword",
"verified_result": null
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/IBM/apigateway-go-sdk v0.0.0-20210714141226-a5d5d49caaca
github.com/IBM/appconfiguration-go-admin-sdk v0.3.0
github.com/IBM/appid-management-go-sdk v0.0.0-20210908164609-dd0e0eaf732f
github.com/IBM/cloud-databases-go-sdk v0.2.0
github.com/IBM/cloud-databases-go-sdk v0.3.0
github.com/IBM/cloudant-go-sdk v0.0.43
github.com/IBM/container-registry-go-sdk v0.0.15
github.com/IBM/continuous-delivery-go-sdk v0.1.2
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ github.com/IBM/appid-management-go-sdk v0.0.0-20210908164609-dd0e0eaf732f h1:4c1
github.com/IBM/appid-management-go-sdk v0.0.0-20210908164609-dd0e0eaf732f/go.mod h1:d22kTYY7RYBWcQlZpqrSdshpB/lJ16viWS5Sbjtlc8s=
github.com/IBM/cloud-databases-go-sdk v0.2.0 h1:OkyYpj1LjUnBEgAkp5LMYyla40keEg4ZzWdEXAig4nk=
github.com/IBM/cloud-databases-go-sdk v0.2.0/go.mod h1:ZOujnMABgw39Mr/5sFd16eniw9mSS4/if1Vht3m+F4M=
github.com/IBM/cloud-databases-go-sdk v0.3.0 h1:yTCfBF05PjBLTuhrwBH5P1onCSls/vbJB2HawBjheOU=
github.com/IBM/cloud-databases-go-sdk v0.3.0/go.mod h1:ZOujnMABgw39Mr/5sFd16eniw9mSS4/if1Vht3m+F4M=
github.com/IBM/cloudant-go-sdk v0.0.43 h1:YxTy4RpAEezX32YIWnds76hrBREmO4u6IkBz1WylNuQ=
github.com/IBM/cloudant-go-sdk v0.0.43/go.mod h1:WeYrJPaHTw19943ndWnVfwMIlZ5z0XUM2uEXNBrwZ1M=
github.com/IBM/container-registry-go-sdk v0.0.15 h1:sfEXm4qNj9ZCwTlFOsdjF5P/lvajU/Sc22yNlzg0F9I=
Expand Down
195 changes: 171 additions & 24 deletions ibm/service/database/resource_ibm_database.go
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ func ResourceIBMDatabaseInstance() *schema.Resource {
Description: "User name",
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(5, 32),
ValidateFunc: validation.StringLenBetween(4, 32),
},
"password": {
Description: "User password",
Expand Down Expand Up @@ -467,6 +467,29 @@ func ResourceIBMDatabaseInstance() *schema.Resource {
},
},
},
"logical_replication_slot": {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should have validation to enforce only postgresql deployments can use logical_replication_slot

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! Took your recommendation and added it here:

if service != "databases-for-postgresql" {
:D

Copy link
Collaborator

@alexhemard alexhemard Oct 27, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we might want to do this in a CustomizeDiff function since this will blow up during runtime rather than during the plan stage

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done :D

Type: schema.TypeSet,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"name": {
Description: "Logical Replication Slot name",
Type: schema.TypeString,
Required: true,
},
"database_name": {
Description: "Logical Replication Slot name",
Type: schema.TypeString,
Required: true,
},
"plugin_type": {
Description: "Logical Replication Slot name",
Type: schema.TypeString,
Required: true,
},
},
},
},
"group": {
Type: schema.TypeSet,
Optional: true,
Expand Down Expand Up @@ -1561,35 +1584,39 @@ func resourceIBMDatabaseInstanceCreate(context context.Context, d *schema.Resour

for _, user := range userList.(*schema.Set).List() {
userEl := user.(map[string]interface{})
createDatabaseUserRequestUserModel := &clouddatabasesv5.User{
Username: core.StringPtr(userEl["name"].(string)),
Password: core.StringPtr(userEl["password"].(string)),
}
if userEl["name"].(string) == "repl" && (strings.Contains(serviceName, "postgresql") || strings.Contains(serviceName, "enterprisedb")) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of a special case for the repl user, we should attempt to update the password if user creation fails in case it already exists i.e. restoring from a backup

updateReplUser(userEl["password"].(string), instanceID, userEl["type"].(string), meta, d)
} else {
createDatabaseUserRequestUserModel := &clouddatabasesv5.User{
Username: core.StringPtr(userEl["name"].(string)),
Password: core.StringPtr(userEl["password"].(string)),
}

// User Role only for ops_manager user type
if userEl["type"].(string) == "ops_manager" && userEl["role"].(string) != "" {
createDatabaseUserRequestUserModel.Role = core.StringPtr(userEl["role"].(string))
}
// User Role only for ops_manager user type
if userEl["type"].(string) == "ops_manager" && userEl["role"].(string) != "" {
createDatabaseUserRequestUserModel.Role = core.StringPtr(userEl["role"].(string))
}

instanceId := d.Id()
createDatabaseUserOptions := &clouddatabasesv5.CreateDatabaseUserOptions{
ID: &instanceId,
UserType: core.StringPtr(userEl["type"].(string)),
User: createDatabaseUserRequestUserModel,
}
instanceId := d.Id()
createDatabaseUserOptions := &clouddatabasesv5.CreateDatabaseUserOptions{
ID: &instanceId,
UserType: core.StringPtr(userEl["type"].(string)),
User: createDatabaseUserRequestUserModel,
}

createDatabaseUserResponse, response, err := cloudDatabasesClient.CreateDatabaseUser(createDatabaseUserOptions)
createDatabaseUserResponse, response, err := cloudDatabasesClient.CreateDatabaseUser(createDatabaseUserOptions)

if err != nil {
return diag.FromErr(fmt.Errorf("CreateDatabaseUser (%s) failed %s\n%s", userEl["name"], err, response))
}
if err != nil {
return diag.FromErr(fmt.Errorf("CreateDatabaseUser (%s) failed %s\n%s", userEl["name"], err, response))
}

taskID := *createDatabaseUserResponse.Task.ID
taskID := *createDatabaseUserResponse.Task.ID

_, err = waitForDatabaseTaskComplete(taskID, d, meta, d.Timeout(schema.TimeoutCreate))
if err != nil {
return diag.FromErr(fmt.Errorf(
"[ERROR] Error waiting for update of database (%s) user (%s) create task to complete: %s", d.Id(), userEl["name"], err))
_, err = waitForDatabaseTaskComplete(taskID, d, meta, d.Timeout(schema.TimeoutCreate))
if err != nil {
return diag.FromErr(fmt.Errorf(
"[ERROR] Error waiting for update of database (%s) user (%s) create task to complete: %s", d.Id(), userEl["name"], err))
}
}
}
}
Expand Down Expand Up @@ -2169,6 +2196,7 @@ func resourceIBMDatabaseInstanceUpdate(context context.Context, d *schema.Resour
}

for _, change := range userChanges {
service := d.Get("service").(string)
// Update Database User password only
if change.Old != nil && change.New != nil {
// No change
Expand Down Expand Up @@ -2209,6 +2237,14 @@ func resourceIBMDatabaseInstanceUpdate(context context.Context, d *schema.Resour
change.Old = nil
}

newName, ok := change.New["name"]
isPgOrEdb := strings.Contains(service, "postgresql") || strings.Contains(service, "enterprisedb")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PG only

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Logical Replication is PG only :D But the repl user exists in both edb and pg. So we need to handle it for both cases.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rather than create a special case for the repl user seems like we should always attempt to update the password first and then create the user if it does not exist.

if change.Old != nil && change.New != nil {

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Smart!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Went ahead and did it!

if ok && isPgOrEdb && newName.(string) == "repl" {
updateReplUser(change.New["password"].(string), instanceID, change.New["type"].(string), meta, d)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if this method is necessary can it be a generic changeUserPassword method?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done :D


continue
}

// Delete Old User
if change.Old != nil {
deleteDatabaseUserOptions := &clouddatabasesv5.DeleteDatabaseUserOptions{
Expand Down Expand Up @@ -2267,6 +2303,85 @@ func resourceIBMDatabaseInstanceUpdate(context context.Context, d *schema.Resour
}
}

if d.HasChange("logical_replication_slot") {
cloudDatabasesClient, err := meta.(conns.ClientSession).CloudDatabasesV5()
if err != nil {
return diag.FromErr(fmt.Errorf("[ERROR] Error getting database client settings: %s", err))
}

oldSlots, newSlots := d.GetChange("logical_replication_slot")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we can't update anything, we don't need to track changes to the state. Something like this may be more straightforwardL

oldList, newList := d.GetChange("logical_replication_slot")
if oldList == nil {
	oldList = new(schema.Set)
}
if newList == nil {
	newList = new(schema.Set)
}
os := oldList.(*schema.Set)
ns := newList.(*schema.Set)
remove := os.Difference(ns).List()
add := ns.Difference(os).List()

if len(add) > 0 {
	for _, slot := range add {
		addSlot := slot.(map[string]interface{})

		logicalReplicationSlot := &clouddatabasesv5.LogicalReplicationSlot{
			Name:         core.StringPtr(addSlot["name"].(string)),
			DatabaseName: core.StringPtr(addSlot["database_name"].(string)),
			PluginType:   core.StringPtr(addSlot["plugin_type"].(string)),
		}

		// ...
	}
}

if len(remove) > 0 {
	for _, slot := range remove {	
		removeSlot := slot.(map[string]interface{})
		
		// ...
	}
}

slotChanges := make(map[string]*userChange)
slotKey := func(raw map[string]interface{}) string {
return fmt.Sprintf("%s-%s", raw["name"].(string), raw["database_name"].(string))
}

for _, raw := range oldSlots.(*schema.Set).List() {
user := raw.(map[string]interface{})
k := slotKey(user)
slotChanges[k] = &userChange{Old: user}
}

for _, raw := range newSlots.(*schema.Set).List() {
user := raw.(map[string]interface{})
k := slotKey(user)
if _, ok := slotChanges[k]; !ok {
slotChanges[k] = &userChange{}
}
slotChanges[k].New = user
}

for _, change := range slotChanges {
// Create New Logical Rep Slot
if change.New != nil {
logicalReplicationSlot := &clouddatabasesv5.LogicalReplicationSlot{
Name: core.StringPtr(change.New["name"].(string)),
DatabaseName: core.StringPtr(change.New["database_name"].(string)),
PluginType: core.StringPtr(change.New["plugin_type"].(string)),
}

createLogicalReplicationOptions := &clouddatabasesv5.CreateLogicalReplicationSlotOptions{
ID: &instanceID,
LogicalReplicationSlot: logicalReplicationSlot,
}

createLogicalRepSlotResponse, response, err := cloudDatabasesClient.CreateLogicalReplicationSlot(createLogicalReplicationOptions)
if err != nil {
return diag.FromErr(fmt.Errorf("[ERROR] CreateLogicalReplicationSlot (%s) failed %s\n%s", *createLogicalReplicationOptions.LogicalReplicationSlot.Name, err, response))
}

taskID := *createLogicalRepSlotResponse.Task.ID
_, err = waitForDatabaseTaskComplete(taskID, d, meta, d.Timeout(schema.TimeoutUpdate))
if err != nil {
return diag.FromErr(fmt.Errorf(
"[ERROR] Error waiting for database (%s) logical replication slot (%s) create task to complete: %s", instanceID, *createLogicalReplicationOptions.LogicalReplicationSlot.Name, err))
}
}

// Delete Old Logical Rep Slot
if change.Old != nil {
deleteDatabaseUserOptions := &clouddatabasesv5.DeleteLogicalReplicationSlotOptions{
ID: &instanceID,
Name: core.StringPtr(change.Old["name"].(string)),
}

deleteDatabaseUserResponse, response, err := cloudDatabasesClient.DeleteLogicalReplicationSlot(deleteDatabaseUserOptions)

if err != nil {
return diag.FromErr(fmt.Errorf(
"[ERROR] DeleteDatabaseUser (%s) failed %s\n%s", *deleteDatabaseUserOptions.Name, err, response))
}

taskID := *deleteDatabaseUserResponse.Task.ID
_, err = waitForDatabaseTaskComplete(taskID, d, meta, d.Timeout(schema.TimeoutUpdate))

if err != nil {
return diag.FromErr(fmt.Errorf(
"[ERROR] Error waiting for database (%s) logical replication slot (%s) delete task to complete: %s", icdId, *deleteDatabaseUserOptions.Name, err))
}
}
}
}

return resourceIBMDatabaseInstanceRead(context, d, meta)
}

Expand Down Expand Up @@ -2939,3 +3054,35 @@ func checkV5Groups(_ context.Context, diff *schema.ResourceDiff, meta interface{

return nil
}

func updateReplUser(password string, instanceID string, userType string, meta interface{}, d *schema.ResourceData) diag.Diagnostics {
cloudDatabasesClient, err := meta.(conns.ClientSession).CloudDatabasesV5()
passwordSettingUser := &clouddatabasesv5.APasswordSettingUser{
Password: core.StringPtr(password),
}

changeUserPasswordOptions := &clouddatabasesv5.ChangeUserPasswordOptions{
ID: &instanceID,
UserType: core.StringPtr(userType),
Username: core.StringPtr("repl"),
User: passwordSettingUser,
}

changeUserPasswordResponse, response, err := cloudDatabasesClient.ChangeUserPassword(changeUserPasswordOptions)

if response.StatusCode != 404 {
if err != nil {
return diag.FromErr(fmt.Errorf("[ERROR] ChangeUserPassword (%s) failed %s\n%s", *changeUserPasswordOptions.Username, err, response))
}

taskID := *changeUserPasswordResponse.Task.ID
_, err = waitForDatabaseTaskComplete(taskID, d, meta, d.Timeout(schema.TimeoutUpdate))

if err != nil {
return diag.FromErr(fmt.Errorf(
"[ERROR] Error waiting for database (%s) user (%s) password update task to complete: %s", instanceID, *changeUserPasswordOptions.Username, err))
}
}

return nil
}
Loading