fix: add backoff to operator retry mechanism

src/pepr/operator/reconcilers/package-reconciler.ts

@@ -35,6 +35,18 @@ export async function packageReconciler(pkg: UDSPackage) {
     return;
   }
 
+  if (pkg.status?.retryAttempt && pkg.status?.retryAttempt > 0) {
+    // calculate exponential backoff where backoffSeconds = 3^retryAttempt
+    const backOffSeconds = 3 ** pkg.status?.retryAttempt;
+
+    log.info(
+      metadata,
+      `Waiting ${backOffSeconds} seconds before processing Package ${namespace}/${name}, status.phase: ${pkg.status?.phase}, observedGeneration: ${pkg.status?.observedGeneration}, retryAttempt: ${pkg.status?.retryAttempt}`,
+    );
+    // wait for backOff seconds before retrying
+    await new Promise(resolve => setTimeout(resolve, backOffSeconds * 1000));
+  }
+
   // Migrate the package to the latest version
   migrate(pkg);