proof: re-try in HasProofs if some are unavailable

When CPU cycles are constrained (e.g. in CI), it can sometimes happen
that we:
 - Import a proof into the multi archiver
 - The multi archiver imports it into each backend
 - The first backend notifies its subscribers after importing
 - The custodian reacts to the notification and checks HasProof on
   the multi archiver
 - Because the other goroutine hasn't finished importing into all
   backends, HasProof returns false
 - The custodian only finishes the transfer once it receives the
   notification from the second backend

All the above leads to the situation where the status "proof received"
is fired twice instead of just once, which causes the itests to fail
sometimes.

Author: guggero

proof/archive.go

@@ -879,21 +879,54 @@ func (m *MultiArchiver) FetchIssuanceProof(ctx context.Context,
 // intended to be a performance optimized lookup compared to fetching a proof
 // and checking for ErrProofNotFound. The multi archiver only considers a proof
 // to be present if all backends have it.
-func (m *MultiArchiver) HasProof(ctx context.Context, id Locator) (bool, error) {
+func (m *MultiArchiver) HasProof(ctx context.Context,
+	id Locator) (bool, error) {
+
+	var (
+		someHaveProof = false
+		allHaveProof  = true
+	)
 	for _, archive := range m.backends {
 		ok, err := archive.HasProof(ctx, id)
 		if err != nil {
 			return false, err
 		}
 
-		// We are expecting all backends to have the proof, otherwise we
-		// consider the proof not to be found.
-		if !ok {
-			return false, nil
-		}
+		someHaveProof = someHaveProof || ok
+		allHaveProof = allHaveProof && ok
+	}
+
+	// If all backends have the proof, then we don't need to do anything
+	// further and can return that result.
+	if allHaveProof {
+		return true, nil
 	}
 
-	return true, nil
+	// If no backends have the proof, then this is just a proof we don't
+	// know about, which is fine too.
+	if !someHaveProof {
+		return false, nil
+	}
+
+	// If only some but not all backends have the proof, it's possible that
+	// the other ones are in the process of importing it right now. So we
+	// re-try a couple of times to see if the proof becomes available
+	// eventually.
+	return fn.RetryFuncN(
+		ctx, fn.DefaultRetryConfig(), func() (bool, error) {
+			allHaveProof = true
+			for _, archive := range m.backends {
+				ok, err := archive.HasProof(ctx, id)
+				if err != nil {
+					return false, err
+				}
+
+				allHaveProof = allHaveProof && ok
+			}
+
+			return allHaveProof, nil
+		},
+	)
 }
 
 // FetchProofs fetches all proofs for assets uniquely identified by the passed