[CSGI-1926] Add delays to API calls retries lacking of them on various TF Objects

pagerduty/data_source_pagerduty_event_orchestration.go

@@ -74,6 +74,9 @@ func dataSourcePagerDutyEventOrchestrationRead(d *schema.ResourceData, meta inte
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 
@@ -96,6 +99,9 @@ func dataSourcePagerDutyEventOrchestrationRead(d *schema.ResourceData, meta inte
 		// since the list ndpoint does not return it
 		orch, _, err := client.EventOrchestrations.Get(found.ID)
 		if err != nil {
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 

pagerduty/data_source_pagerduty_event_orchestrations.go

@@ -91,6 +91,7 @@ func dataSourcePagerDutyEventOrchestrationsRead(d *schema.ResourceData, meta int
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		}
 
@@ -125,6 +126,7 @@ func dataSourcePagerDutyEventOrchestrationsRead(d *schema.ResourceData, meta int
 					return resource.NonRetryableError(err)
 				}
 
+				time.Sleep(10 * time.Second)
 				return resource.RetryableError(err)
 			}
 			orchestrations = append(orchestrations, orch)

pagerduty/resource_pagerduty_automation_actions_action.go

@@ -344,6 +344,9 @@ func resourcePagerDutyAutomationActionsActionDelete(d *schema.ResourceData, meta
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_automation_actions_action_service_association.go

@@ -120,6 +120,9 @@ func resourcePagerDutyAutomationActionsActionServiceAssociationDelete(d *schema.
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_automation_actions_action_team_association.go

@@ -124,6 +124,9 @@ func resourcePagerDutyAutomationActionsActionTeamAssociationDelete(d *schema.Res
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_automation_actions_runner.go

@@ -206,6 +206,9 @@ func resourcePagerDutyAutomationActionsRunnerDelete(d *schema.ResourceData, meta
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_automation_actions_runner_team_association.go

@@ -120,6 +120,9 @@ func resourcePagerDutyAutomationActionsRunnerTeamAssociationDelete(d *schema.Res
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_business_service.go

@@ -107,6 +107,9 @@ func resourcePagerDutyBusinessServiceCreate(d *schema.ResourceData, meta interfa
 		}
 		log.Printf("[INFO] Creating PagerDuty business service %s", businessService.Name)
 		if businessService, _, err = client.BusinessServices.Create(businessService); err != nil {
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		} else if businessService != nil {
 			d.SetId(businessService.ID)
@@ -135,6 +138,9 @@ func resourcePagerDutyBusinessServiceRead(d *schema.ResourceData, meta interface
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		} else if businessService != nil {
 			d.Set("name", businessService.Name)

pagerduty/resource_pagerduty_business_service_subscriber.go

@@ -70,6 +70,9 @@ func resourcePagerDutyBusinessServiceSubscriberCreate(d *schema.ResourceData, me
 
 		log.Printf("[INFO] Creating PagerDuty business service %s subscriber %s type %s", businessServiceId, businessServiceSubscriber.ID, businessServiceSubscriber.Type)
 		if _, err = client.BusinessServiceSubscribers.Create(businessServiceId, businessServiceSubscriber); err != nil {
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		} else if businessServiceSubscriber != nil {
 			// create subscriber assignment it as PagerDuty API does not return one

pagerduty/resource_pagerduty_escalation_policy.go

@@ -198,6 +198,9 @@ func resourcePagerDutyEscalationPolicyUpdate(d *schema.ResourceData, meta interf
 
 	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if _, _, err := client.EscalationPolicies.Update(d.Id(), escalationPolicy); err != nil {
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil
@@ -222,6 +225,7 @@ func resourcePagerDutyEscalationPolicyDelete(d *schema.ResourceData, meta interf
 	retryErr := resource.Retry(30*time.Second, func() *resource.RetryError {
 		if _, err := client.EscalationPolicies.Delete(d.Id()); err != nil {
 			if isErrCode(err, 400) {
+				time.Sleep(10 * time.Second)
 				return resource.RetryableError(err)
 			}
 

pagerduty/resource_pagerduty_event_orchestration.go

@@ -110,6 +110,7 @@ func resourcePagerDutyEventOrchestrationCreate(d *schema.ResourceData, meta inte
 	retryErr := resource.Retry(10*time.Second, func() *resource.RetryError {
 		if orch, _, err := client.EventOrchestrations.Create(payload); err != nil {
 			if isErrCode(err, 400) || isErrCode(err, 429) {
+				time.Sleep(2 * time.Second)
 				return resource.RetryableError(err)
 			}
 
@@ -171,6 +172,7 @@ func resourcePagerDutyEventOrchestrationUpdate(d *schema.ResourceData, meta inte
 	retryErr := resource.Retry(10*time.Second, func() *resource.RetryError {
 		if _, _, err := client.EventOrchestrations.Update(d.Id(), orchestration); err != nil {
 			if isErrCode(err, 400) || isErrCode(err, 429) {
+				time.Sleep(2 * time.Second)
 				return resource.RetryableError(err)
 			}
 			return resource.NonRetryableError(err)

pagerduty/resource_pagerduty_event_orchestration_integration.go

@@ -120,10 +120,14 @@ func resourcePagerDutyEventOrchestrationIntegrationCreate(ctx context.Context, d
 			if isErrCode(err, 400) {
 				return resource.NonRetryableError(err)
 			}
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		} else if integration != nil {
 			// Try reading an integration after creation, retry if not found:
 			if _, readErr := fetchPagerDutyEventOrchestrationIntegration(ctx, d, meta, oid, integration.ID, true); readErr != nil {
+				time.Sleep(30 * time.Second)
 				log.Printf("[WARN] Cannot locate Integration '%s' on PagerDuty Event Orchestration '%s'. Retrying creation...", integration.ID, oid)
 				return resource.RetryableError(readErr)
 			}
@@ -151,6 +155,9 @@ func resourcePagerDutyEventOrchestrationIntegrationRead(ctx context.Context, d *
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 
@@ -185,6 +192,9 @@ func resourcePagerDutyEventOrchestrationIntegrationUpdate(ctx context.Context, d
 				if isErrCode(err, 400) {
 					return resource.NonRetryableError(err)
 				}
+				// Delaying retry by 30s as recommended by PagerDuty
+				// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+				time.Sleep(30 * time.Second)
 				return resource.RetryableError(err)
 			} else {
 				// try reading the migrated integration from destination and source:
@@ -193,12 +203,14 @@ func resourcePagerDutyEventOrchestrationIntegrationUpdate(ctx context.Context, d
 
 				// retry migration if the read request returned an error:
 				if readDestErr != nil {
+					time.Sleep(30 * time.Second)
 					log.Printf("[WARN] Integration '%s' cannot be found on the destination PagerDuty Event Orchestration '%s'. Retrying migration....", id, destinationOrchId)
 					return resource.RetryableError(readDestErr)
 				}
 
 				// retry migration if the integration still exists on the source:
 				if readSrcErr == nil && srcInt != nil {
+					time.Sleep(30 * time.Second)
 					log.Printf("[WARN] Integration '%s' still exists on the source PagerDuty Event Orchestration '%s'. Retrying migration....", id, sourceOrchId)
 					return resource.RetryableError(fmt.Errorf("Integration '%s' still exists on the source PagerDuty Event Orchestration '%s'.", id, sourceOrchId))
 				}
@@ -227,6 +239,9 @@ func resourcePagerDutyEventOrchestrationIntegrationUpdate(ctx context.Context, d
 			} else if integration != nil {
 				// Try reading an integration after updating the label, retry if the label is not updated:
 				if updInt, readErr := fetchPagerDutyEventOrchestrationIntegration(ctx, d, meta, oid, id, true); readErr != nil && updInt != nil {
+					// Delaying retry by 30s as recommended by PagerDuty
+					// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+					time.Sleep(30 * time.Second)
 					log.Printf("[WARN] Label for Integration '%s' on PagerDuty Event Orchestration '%s' was not updated. Expected: '%s', actual: '%s'. Retrying update...", id, oid, payload.Label, updInt.Label)
 					return resource.RetryableError(readErr)
 				}
@@ -259,10 +274,14 @@ func resourcePagerDutyEventOrchestrationIntegrationDelete(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		} else {
 			// Try reading an integration after deletion, retry if still found:
 			if integr, _, readErr := client.EventOrchestrationIntegrations.GetContext(ctx, oid, id); readErr == nil && integr != nil {
+				time.Sleep(30 * time.Second)
 				log.Printf("[WARN] Integration '%s' still exists on PagerDuty Event Orchestration '%s'. Retrying deletion...", id, oid)
 				return resource.RetryableError(fmt.Errorf("Integration '%s' still exists on PagerDuty Event Orchestration '%s'.", id, oid))
 			}
@@ -295,6 +314,9 @@ func resourcePagerDutyEventOrchestrationIntegrationImport(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_event_orchestration_path_global.go

@@ -220,6 +220,7 @@ func resourcePagerDutyEventOrchestrationPathGlobalUpdate(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		} else if response != nil {
 			d.SetId(response.OrchestrationPath.Parent.ID)
@@ -257,6 +258,7 @@ func resourcePagerDutyEventOrchestrationPathGlobalDelete(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_event_orchestration_path_router.go

@@ -176,6 +176,7 @@ func resourcePagerDutyEventOrchestrationPathRouterDelete(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil
@@ -209,6 +210,7 @@ func resourcePagerDutyEventOrchestrationPathRouterUpdate(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		}
 		if response == nil {

pagerduty/resource_pagerduty_event_orchestration_path_service.go

@@ -266,6 +266,7 @@ func resourcePagerDutyEventOrchestrationPathServiceUpdate(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		} else if response != nil {
 			d.SetId(response.OrchestrationPath.Parent.ID)
@@ -349,6 +350,7 @@ func resourcePagerDutyEventOrchestrationPathServiceDelete(ctx context.Context, d
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil

pagerduty/resource_pagerduty_event_orchestration_path_unrouted.go

@@ -226,6 +226,7 @@ func resourcePagerDutyEventOrchestrationPathUnroutedDelete(ctx context.Context,
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil
@@ -259,6 +260,7 @@ func resourcePagerDutyEventOrchestrationPathUnroutedUpdate(ctx context.Context,
 				return resource.NonRetryableError(err)
 			}
 
+			time.Sleep(10 * time.Second)
 			return resource.RetryableError(err)
 		}
 

pagerduty/resource_pagerduty_schedule.go

@@ -372,6 +372,9 @@ func resourcePagerDutyScheduleUpdate(d *schema.ResourceData, meta interface{}) e
 
 	retryErr := resource.Retry(2*time.Minute, func() *resource.RetryError {
 		if _, _, err := client.Schedules.Update(d.Id(), schedule, opts); err != nil {
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil
@@ -422,6 +425,9 @@ func resourcePagerDutyScheduleDelete(d *schema.ResourceData, meta interface{}) e
 	retryErr = resource.Retry(2*time.Minute, func() *resource.RetryError {
 		if _, err := client.Schedules.Delete(scheduleId); err != nil {
 			if !isErrCode(err, 400) {
+				// Delaying retry by 30s as recommended by PagerDuty
+				// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+				time.Sleep(30 * time.Second)
 				return resource.RetryableError(err)
 			}
 			isErrorScheduleUsedByEP := func(e *pagerduty.Error) bool {
@@ -464,6 +470,9 @@ func resourcePagerDutyScheduleDelete(d *schema.ResourceData, meta interface{}) e
 			epsDataUsingThisSchedule, errFetchingFullEPs := fetchEPsDataUsingASchedule(epsUsingThisSchedule, client)
 			if errFetchingFullEPs != nil {
 				err = fmt.Errorf("%v; %w", err, errFetchingFullEPs)
+				// Delaying retry by 30s as recommended by PagerDuty
+				// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+				time.Sleep(30 * time.Second)
 				return resource.RetryableError(err)
 			}
 
@@ -478,6 +487,9 @@ func resourcePagerDutyScheduleDelete(d *schema.ResourceData, meta interface{}) e
 			if workaroundErr != nil {
 				err = fmt.Errorf("%v; %w", err, workaroundErr)
 			}
+			// Delaying retry by 30s as recommended by PagerDuty
+			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+			time.Sleep(30 * time.Second)
 			return resource.RetryableError(err)
 		}
 		return nil
@@ -764,6 +776,7 @@ func removeScheduleFromEP(c *pagerduty.Client, scheduleID string, ep *pagerduty.
 		_, _, err := c.EscalationPolicies.Update(ep.ID, ep)
 		if err != nil {
 			if !isErrCode(err, 404) {
+				time.Sleep(2 * time.Second)
 				return resource.RetryableError(err)
 			}
 		}
@@ -858,6 +871,7 @@ func fetchEPsDataUsingASchedule(eps []string, c *pagerduty.Client) ([]*pagerduty
 					return resource.NonRetryableError(err)
 				}
 
+				time.Sleep(2 * time.Second)
 				return resource.RetryableError(err)
 			}
 			fullEPs = append(fullEPs, ep)

pagerduty/resource_pagerduty_service_dependency.go

@@ -157,6 +157,9 @@ func resourcePagerDutyServiceDependencyAssociate(ctx context.Context, d *schema.
 
 		if err != nil {
 			if isErrCode(err, 404) {
+				// Delaying retry by 30s as recommended by PagerDuty
+				// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+				time.Sleep(30 * time.Second)
 				return resource.RetryableError(err)
 			}
 			return resource.NonRetryableError(err)
@@ -202,7 +205,6 @@ func resourcePagerDutyServiceDependencyDisassociate(ctx context.Context, d *sche
 			// Delaying retry by 30s as recommended by PagerDuty
 			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
 			time.Sleep(30 * time.Second)
-
 			return resource.RetryableError(err)
 		} else if dependencies != nil {
 			for _, rel := range dependencies.Relationships {
@@ -247,7 +249,6 @@ func resourcePagerDutyServiceDependencyDisassociate(ctx context.Context, d *sche
 			// Delaying retry by 30s as recommended by PagerDuty
 			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
 			time.Sleep(30 * time.Second)
-
 			return resource.RetryableError(err)
 		}
 		return nil
@@ -326,7 +327,6 @@ func findDependencySetState(ctx context.Context, depID, serviceID, serviceType s
 			// Delaying retry by 30s as recommended by PagerDuty
 			// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
 			time.Sleep(30 * time.Second)
-
 			return resource.RetryableError(err)
 		} else if dependencies != nil {
 			depFound := false