fix: Treat 502s and 503s as transient for GCS OS (#2202)

A refactoring introduced lately caused multiple restarts in provers
(namely BWGs) when GCS was unavailable (502 or 503). This is a sporadic,
once in a while, but still invalides tens of minutes of work and makes
proving fickle and slow. This PR addresses the issue and restores old
behavior pre-refactoring, treating 502s and 503s as transient errors.

core/lib/object_store/Cargo.toml

@@ -26,6 +26,7 @@ rand.workspace = true
 tokio = { workspace = true, features = ["full"] }
 tracing.workspace = true
 prost.workspace = true
+reqwest.workspace = true
 
 [dev-dependencies]
 assert_matches.workspace = true

core/lib/object_store/src/gcs.rs

@@ -87,7 +87,7 @@ impl From<AuthError> for ObjectStoreError {
     fn from(err: AuthError) -> Self {
         let is_transient = matches!(
             &err,
-            AuthError::HttpError(err) if err.is_timeout() || err.is_connect() || has_transient_io_source(err)
+            AuthError::HttpError(err) if err.is_timeout() || err.is_connect() || has_transient_io_source(err) || upstream_unavailable(err)
         );
         Self::Initialization {
             source: err.into(),
@@ -111,6 +111,11 @@ fn has_transient_io_source(mut err: &(dyn StdError + 'static)) -> bool {
     }
 }
 
+fn upstream_unavailable(err: &reqwest::Error) -> bool {
+    err.status() == Some(StatusCode::BAD_GATEWAY)
+        || err.status() == Some(StatusCode::SERVICE_UNAVAILABLE)
+}
+
 impl From<HttpError> for ObjectStoreError {
     fn from(err: HttpError) -> Self {
         let is_not_found = match &err {
@@ -126,7 +131,7 @@ impl From<HttpError> for ObjectStoreError {
         } else {
             let is_transient = matches!(
                 &err,
-                HttpError::HttpClient(err) if err.is_timeout() || err.is_connect() || has_transient_io_source(err)
+                HttpError::HttpClient(err) if err.is_timeout() || err.is_connect() || has_transient_io_source(err) || upstream_unavailable(err)
             );
             ObjectStoreError::Other {
                 is_transient,