Make staging repository operations retry on failure (#17)

* Make staging repository operations retry on failure

* Minimize diff

* Keep the former HttpClientUtil.create signature

So that it still throws when something goes wrong, and users aren't
surprised with the new behavior of createResponse

* Increase duration between attempts via exponential smoothing

* Fix package name

* fixup Increase duration between attempts via exponential smoothing

* fixup Fix package name

---------

Co-authored-by: Alexandre Archambault <alexandre.archambault@gmail.com>

publish/src/coursier/publish/sonatype/HttpClientUtil.scala

@@ -57,17 +57,25 @@ private[sonatype] final case class HttpClientUtil(
   }
 
   def create(url: String, post: Option[Array[Byte]] = None, isJson: Boolean = false): Unit = {
+    val resp = createResponse(url, post, isJson)
+    if (resp.code.code != 201)
+      throw new Exception(
+        s"Failed to get $url (http status: ${resp.code.code}, response: ${Try(new String(resp.body, StandardCharsets.UTF_8)).getOrElse("")})"
+      )
+  }
 
+  def createResponse(
+    url: String,
+    post: Option[Array[Byte]] = None,
+    isJson: Boolean = false
+  ): Response[Array[Byte]] = {
     if (verbosity >= 1)
       Console.err.println(s"Getting $url")
     val resp = request(url, post, isJson).send(backend)
     if (verbosity >= 1)
-      Console.err.println(s"Done: $url")
+      Console.err.println(s"Got HTTP ${resp.code.code} from $url")
 
-    if (resp.code.code != 201)
-      throw new Exception(
-        s"Failed to get $url (http status: ${resp.code.code}, response: ${Try(new String(resp.body, StandardCharsets.UTF_8)).getOrElse("")})"
-      )
+    resp
   }
 
   def get[T: JsonValueCodec](

publish/src/coursier/publish/sonatype/SonatypeApi.scala

@@ -7,18 +7,22 @@ import com.github.plokhotnyuk.jsoniter_scala.core._
 import com.github.plokhotnyuk.jsoniter_scala.macros._
 import coursier.core.Authentication
 import coursier.publish.sonatype.logger.SonatypeLogger
+import coursier.publish.util.EmaRetryParams
 import coursier.util.Task
 import sttp.client3._
 
-import scala.concurrent.duration.Duration
+import scala.annotation.tailrec
+import scala.concurrent.duration.{Duration, DurationInt}
+import scala.util.Try
 import scala.util.control.NonFatal
 
 final case class SonatypeApi(
   backend: SttpBackend[Identity, Any],
   base: String,
   authentication: Option[Authentication],
   verbosity: Int,
-  retryOnTimeout: Int = 3
+  retryOnTimeout: Int = 3,
+  stagingRepoRetryParams: EmaRetryParams = EmaRetryParams(3, 10.seconds.toMillis, 2.0f)
 ) {
 
   // vaguely inspired by https://github.com/lihaoyi/mill/blob/7b4ced648ecd9b79b3a16d67552f0bb69f4dd543/scalalib/src/mill/scalalib/publish/SonatypeHttpApi.scala
@@ -125,12 +129,26 @@ final case class SonatypeApi(
     profile: Profile,
     repositoryId: String,
     description: String
-  ): Unit =
-    clientUtil.create(
-      s"${profile.uri}/$action",
-      post = Some(postBody(writeToArray(StagedRepositoryRequest(description, repositoryId)))),
-      isJson = true
-    )
+  ): Unit = {
+    val url  = s"${profile.uri}/$action"
+    val body = postBody(writeToArray(StagedRepositoryRequest(description, repositoryId)))
+    @tailrec
+    def sendRequest(attempt: Int, waitDurationMs: Long): Unit = {
+      val resp = clientUtil.createResponse(url, post = Some(body), isJson = true)
+
+      if (!resp.code.isSuccess) {
+        if (attempt >= stagingRepoRetryParams.attempts)
+          throw new Exception(
+            s"Failed to get $url (http status: ${resp.code.code}, response: ${Try(new String(resp.body, StandardCharsets.UTF_8)).getOrElse("")})"
+          )
+
+        Thread.sleep(waitDurationMs)
+        sendRequest(attempt + 1, stagingRepoRetryParams.update(waitDurationMs))
+      }
+    }
+
+    sendRequest(1, stagingRepoRetryParams.initialWaitDurationMs)
+  }
 
   def sendCloseStagingRepositoryRequest(
     profile: Profile,

publish/src/coursier/publish/util/EmaRetryParams.scala

@@ -0,0 +1,12 @@
+package coursier.publish.util
+
+import scala.math.Numeric.Implicits
+
+final case class EmaRetryParams(
+  attempts: Int,
+  initialWaitDurationMs: Long,
+  factor: Float
+) {
+  def update(value: Long): Long =
+    math.ceil(value * factor.toDouble).toLong
+}

publish/test/src/coursier/publish/sonatype/SonatypeTests.scala

@@ -0,0 +1,63 @@
+package coursier.publish.sonatype
+
+import coursier.publish.util.EmaRetryParams
+import sttp.client3.testing.SttpBackendStub
+
+import utest._
+
+object SonatypeTests extends TestSuite {
+  val tests = Tests {
+    test("Retry sonatype repository actions") {
+      var count = 0
+      val mockBackend = SttpBackendStub.synchronous
+        .whenRequestMatches { _ => count += 1; count < 6 }
+        .thenRespondServerError()
+        .whenRequestMatches(_ => count >= 6)
+        .thenRespondOk()
+
+      {
+        val sonatypeApi20 = SonatypeApi(
+          mockBackend,
+          base = "https://oss.sonatype.org",
+          authentication = None,
+          verbosity = 0,
+          retryOnTimeout = 1,
+          stagingRepoRetryParams = EmaRetryParams(20, 100L, 1.0f)
+        )
+
+        sonatypeApi20.sendPromoteStagingRepositoryRequest(
+          SonatypeApi.Profile("id", "name", "uri"),
+          "repo",
+          "description"
+        )
+
+        assert(count == 6)
+      }
+
+      count = 0
+
+      {
+        val sonatypeApi3 = SonatypeApi(
+          mockBackend,
+          base = "https://oss.sonatype.org",
+          authentication = None,
+          verbosity = 0,
+          retryOnTimeout = 1
+        )
+
+        try sonatypeApi3.sendPromoteStagingRepositoryRequest(
+          SonatypeApi.Profile("id", "name", "uri"),
+          "repo",
+          "description"
+        )
+        catch {
+          case e: Exception
+              if e.getMessage == "Failed to get uri/promote (http status: 500, response: Internal server error)" =>
+          case _: Throwable => assert(false)
+        }
+
+        assert(count == 3)
+      }
+    }
+  }
+}