From ad11ea539666d0d402d3a0a638ca122c4db30237 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 7 Aug 2024 08:24:24 -0500 Subject: [PATCH 1/6] chore: add backoff to operator retry mechanism --- .../operator/reconcilers/package-reconciler.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 0312441ed..d440e94e9 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -35,6 +35,22 @@ export async function packageReconciler(pkg: UDSPackage) { return; } + if (pkg.status?.retryAttempt && pkg.status?.retryAttempt > 0) { + // calculate exponential backoff where backoff = 3s * 2^retryAttempt + const backOff = 3000 * (2 ** (pkg.status?.retryAttempt ?? 0)); + + log.info( + metadata, + `Waiting ${ + backOff / 1000 + } seconds before processing Package ${namespace}/${name}, status.phase: ${pkg.status + ?.phase}, observedGeneration: ${pkg.status?.observedGeneration}, retryAttempt: ${pkg.status + ?.retryAttempt}`, + ); + // wait for backOff seconds before retrying + await new Promise(resolve => setTimeout(resolve, backOff)); + } + // Migrate the package to the latest version migrate(pkg); From c7bdd23417affc303bbd2cd8668a620835616ec7 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 7 Aug 2024 08:38:29 -0500 Subject: [PATCH 2/6] pepr format --- src/pepr/operator/reconcilers/package-reconciler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index d440e94e9..73170abde 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -37,7 +37,7 @@ export async function packageReconciler(pkg: UDSPackage) { if (pkg.status?.retryAttempt && pkg.status?.retryAttempt > 0) { // calculate exponential backoff where backoff = 3s * 2^retryAttempt - const backOff = 3000 * (2 ** (pkg.status?.retryAttempt ?? 0)); + const backOff = 3000 * 2 ** (pkg.status?.retryAttempt ?? 0); log.info( metadata, From 0ee9131ea73df8a030925243cbb7af63961ae317 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 7 Aug 2024 09:00:48 -0500 Subject: [PATCH 3/6] backoff math fix --- src/pepr/operator/reconcilers/package-reconciler.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 73170abde..92688951f 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -36,19 +36,19 @@ export async function packageReconciler(pkg: UDSPackage) { } if (pkg.status?.retryAttempt && pkg.status?.retryAttempt > 0) { - // calculate exponential backoff where backoff = 3s * 2^retryAttempt - const backOff = 3000 * 2 ** (pkg.status?.retryAttempt ?? 0); + // calculate exponential backoff where backoffSeconds = 3^retryAttempt + const backOffSeconds = 3 ** (pkg.status?.retryAttempt); log.info( metadata, `Waiting ${ - backOff / 1000 + backOffSeconds } seconds before processing Package ${namespace}/${name}, status.phase: ${pkg.status ?.phase}, observedGeneration: ${pkg.status?.observedGeneration}, retryAttempt: ${pkg.status ?.retryAttempt}`, ); // wait for backOff seconds before retrying - await new Promise(resolve => setTimeout(resolve, backOff)); + await new Promise(resolve => setTimeout(resolve, backOffSeconds * 1000)); } // Migrate the package to the latest version From 493925b79a9cbd226537f7ef55e6a59a87a6b700 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 7 Aug 2024 09:16:58 -0500 Subject: [PATCH 4/6] pepr format --- src/pepr/operator/reconcilers/package-reconciler.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 92688951f..768d2ffb1 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -37,15 +37,11 @@ export async function packageReconciler(pkg: UDSPackage) { if (pkg.status?.retryAttempt && pkg.status?.retryAttempt > 0) { // calculate exponential backoff where backoffSeconds = 3^retryAttempt - const backOffSeconds = 3 ** (pkg.status?.retryAttempt); + const backOffSeconds = 3 ** pkg.status?.retryAttempt; log.info( metadata, - `Waiting ${ - backOffSeconds - } seconds before processing Package ${namespace}/${name}, status.phase: ${pkg.status - ?.phase}, observedGeneration: ${pkg.status?.observedGeneration}, retryAttempt: ${pkg.status - ?.retryAttempt}`, + `Waiting ${backOffSeconds} seconds before processing Package ${namespace}/${name}, status.phase: ${pkg.status?.phase}, observedGeneration: ${pkg.status?.observedGeneration}, retryAttempt: ${pkg.status?.retryAttempt}`, ); // wait for backOff seconds before retrying await new Promise(resolve => setTimeout(resolve, backOffSeconds * 1000)); From 7532c248142fa866d9ef80e325358fb648fc94ba Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 7 Aug 2024 14:40:44 -0500 Subject: [PATCH 5/6] add event for retry delay --- src/pepr/operator/reconcilers/package-reconciler.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 768d2ffb1..aace52a2e 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -1,4 +1,4 @@ -import { handleFailure, shouldSkip, updateStatus } from "."; +import { handleFailure, shouldSkip, updateStatus, writeEvent } from "."; import { UDSConfig } from "../../config"; import { Component, setupLogger } from "../../logger"; import { enableInjection } from "../controllers/istio/injection"; @@ -41,8 +41,13 @@ export async function packageReconciler(pkg: UDSPackage) { log.info( metadata, - `Waiting ${backOffSeconds} seconds before processing Package ${namespace}/${name}, status.phase: ${pkg.status?.phase}, observedGeneration: ${pkg.status?.observedGeneration}, retryAttempt: ${pkg.status?.retryAttempt}`, + `Waiting ${backOffSeconds} seconds before processing package ${namespace}/${name}, status.phase: ${pkg.status?.phase}, observedGeneration: ${pkg.status?.observedGeneration}, retryAttempt: ${pkg.status?.retryAttempt}`, ); + + await writeEvent(pkg, { + message: `Waiting ${backOffSeconds} seconds before retrying package ${namespace}/${name}`, + }); + // wait for backOff seconds before retrying await new Promise(resolve => setTimeout(resolve, backOffSeconds * 1000)); } From 13e81af167bbe97119aebb819043c821cc8f3aa2 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Thu, 8 Aug 2024 10:52:28 -0500 Subject: [PATCH 6/6] remove package name from event --- src/pepr/operator/reconcilers/package-reconciler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index aace52a2e..568f71afa 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -45,7 +45,7 @@ export async function packageReconciler(pkg: UDSPackage) { ); await writeEvent(pkg, { - message: `Waiting ${backOffSeconds} seconds before retrying package ${namespace}/${name}`, + message: `Waiting ${backOffSeconds} seconds before retrying package`, }); // wait for backOff seconds before retrying