From 2c47480de00121e16216815e37e0c2e5ae625dad Mon Sep 17 00:00:00 2001
From: Parker Selbert <parker@sorentwo.com>
Date: Fri, 16 Aug 2024 08:26:20 -0500
Subject: [PATCH] Increase xact retry delay and increase per attempt

Bump the base transaction retry from 100ms to 500ms, and increase
linearly between each successive attempt to provide deeper backoff. This
alleviates pressure on smaller connection pools and gives more time to
recover from contentions failures.
---
 lib/oban/peers/postgres.ex | 12 +++++++-----
 lib/oban/repo.ex           | 10 +++++-----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/lib/oban/peers/postgres.ex b/lib/oban/peers/postgres.ex
index e9274e67..b94f9f77 100644
--- a/lib/oban/peers/postgres.ex
+++ b/lib/oban/peers/postgres.ex
@@ -70,11 +70,13 @@ defmodule Oban.Peers.Postgres do
     if is_reference(timer), do: Process.cancel_timer(timer)
 
     if state.leader? do
+      fun = fn ->
+        delete_self(state)
+        notify_down(state)
+      end
+
       try do
-        Repo.transaction(state.conf, fn ->
-          delete_self(state)
-          notify_down(state)
-        end)
+        Repo.transaction(state.conf, fun, retry: 1)
       catch
         :exit, _reason -> :ok
       end
@@ -102,7 +104,7 @@ defmodule Oban.Peers.Postgres do
           |> upsert_peer()
         end
 
-        case Repo.transaction(state.conf, fun) do
+        case Repo.transaction(state.conf, fun, retry: 1) do
           {:ok, state} ->
             {state, %{meta | leader: state.leader?}}
 
diff --git a/lib/oban/repo.ex b/lib/oban/repo.ex
index 3c8c09b7..d8f935d5 100644
--- a/lib/oban/repo.ex
+++ b/lib/oban/repo.ex
@@ -64,7 +64,7 @@ defmodule Oban.Repo do
     update_all: 3
   ]
 
-  @retry_opts delay: 100, retry: 5, expected_delay: 5, expected_retry: 10
+  @retry_opts delay: 500, retry: 5, expected_delay: 10, expected_retry: 20
 
   for {fun, arity} <- @callbacks_without_opts do
     args = [Macro.var(:conf, __MODULE__) | Macro.generate_arguments(arity, __MODULE__)]
@@ -141,11 +141,11 @@ defmodule Oban.Repo do
 
   Backoff helpers, in addition to the standard transaction options:
 
-  * `delay` — the time to sleep between retries, defaults to `100ms`
+  * `delay` — the time to sleep between retries, defaults to `500ms`
   * `retry` — the number of retries for unexpected errors, defaults to `5`
   * `expected_delay` — the time to sleep between expected errors, e.g. `serialization` or
-    `lock_not_available`, defaults to `5ms`
-  * `expected_retry` — the number of retries for expected errors, defaults to `10`
+    `lock_not_available`, defaults to `10ms`
+  * `expected_retry` — the number of retries for expected errors, defaults to `20`
   """
   @doc since: "2.18.1"
   def transaction(conf, fun_or_multi, opts \\ []) do
@@ -163,7 +163,7 @@ defmodule Oban.Repo do
           jittery_sleep(opts[:expected_delay])
 
         attempt < opts[:retry] ->
-          jittery_sleep(opts[:delay])
+          jittery_sleep(attempt * opts[:delay])
 
         true ->
           reraise error, __STACKTRACE__