add retries to mobilecoind when it submits transactions (#3308)

* add retries to mobilecoind when it submits transactions

i'm finding that sometimes it fails with attestation permission
denied errors otherwise

---

feel free to leave comments about how you would like this to be
more configurable or use exponential backoff or something, let's
decide exactly how it should work during review

* make it match full-service retries

* tweak retry timings per review

* unused import

* fix build

* clippy

mobilecoind/src/payments.rs

@@ -36,12 +36,12 @@ use rand::Rng;
 use std::{
     cmp::{max, Reverse},
     collections::BTreeMap,
-    iter::empty,
     str::FromStr,
     sync::{
         atomic::{AtomicUsize, Ordering},
         Arc,
     },
+    time::Duration,
 };
 
 /// Default number of blocks used for calculating transaction tombstone block
@@ -826,12 +826,22 @@ impl<T: BlockchainConnection + UserTxConnection + 'static, FPR: FogPubkeyResolve
         let idx = self.submit_node_offset.fetch_add(1, Ordering::SeqCst);
         let responder_id = &responder_ids[idx % responder_ids.len()];
 
+        // The rationale for these retries is:
+        // * Quite often, the attested connetion was closed and we need to retry once to
+        //   re-attest, and this attestation is successful. So that takes 50 ms.
+        // * After that, we back off to 500 ms, because there are rate limits of about
+        //   100 / min in place in production.
+        let retry_iterator = [50, 500, 500, 750, 1000]
+            .iter()
+            .cloned()
+            .map(Duration::from_millis);
+
         // Try and submit.
         let block_height = self
             .peer_manager
             .conn(responder_id)
             .ok_or(Error::NodeNotFound)?
-            .propose_tx(&tx_proposal.tx, empty())
+            .propose_tx(&tx_proposal.tx, retry_iterator)
             .map_err(Error::from)?;
 
         log::info!(