apply timeouts to rewards tree downloads

shared/services/rewards/utils.go

@@ -273,73 +273,83 @@ func (i *IntervalInfo) DownloadRewardsFile(cfg *config.RocketPoolConfig, isDaemo
 
 	// Attempt downloads
 	errBuilder := strings.Builder{}
-	for _, url := range urls {
-		resp, err := http.Get(url)
-		if err != nil {
-			errBuilder.WriteString(fmt.Sprintf("Downloading %s failed (%s)\n", url, err.Error()))
-			continue
+	// ipfs http services are very unreliable and like to hold the connection open for several
+	// minutes before returning a 504. Force a short timeout, but if all sources fail,
+	// gradually increase the timeout to be unreasonably long.
+	for _, timeout := range []time.Duration{200 * time.Millisecond, 2 * time.Second, 60 * time.Second} {
+		client := http.Client{
+			Timeout: timeout,
 		}
-		defer resp.Body.Close()
+		for _, url := range urls {
+			resp, err := client.Get(url)
+			if err != nil {
+				errBuilder.WriteString(fmt.Sprintf("Downloading %s failed (%s)\n", url, err.Error()))
+				continue
+			}
+			defer resp.Body.Close()
 
-		if resp.StatusCode != http.StatusOK {
-			errBuilder.WriteString(fmt.Sprintf("Downloading %s failed with status %s\n", url, resp.Status))
-			continue
-		}
-		// If we got here, we have a successful download
-		bytes, err := io.ReadAll(resp.Body)
-		if err != nil {
-			errBuilder.WriteString(fmt.Sprintf("Error reading response bytes from %s: %s\n", url, err.Error()))
-			continue
-		}
-		writeBytes := bytes
-		if strings.HasSuffix(url, config.RewardsTreeIpfsExtension) {
-			// Decompress it
-			writeBytes, err = decompressFile(bytes)
+			if resp.StatusCode != http.StatusOK {
+				errBuilder.WriteString(fmt.Sprintf("Downloading %s failed with status %s\n", url, resp.Status))
+				continue
+			}
+			// If we got here, we have a successful download
+			bytes, err := io.ReadAll(resp.Body)
 			if err != nil {
-				errBuilder.WriteString(fmt.Sprintf("Error decompressing %s: %s\n", url, err.Error()))
+				errBuilder.WriteString(fmt.Sprintf("Error reading response bytes from %s: %s\n", url, err.Error()))
 				continue
 			}
-		}
+			writeBytes := bytes
+			if strings.HasSuffix(url, config.RewardsTreeIpfsExtension) {
+				// Decompress it
+				writeBytes, err = decompressFile(bytes)
+				if err != nil {
+					errBuilder.WriteString(fmt.Sprintf("Error decompressing %s: %s\n", url, err.Error()))
+					continue
+				}
+			}
 
-		deserializedRewardsFile, err := DeserializeRewardsFile(writeBytes)
-		if err != nil {
-			return fmt.Errorf("Error deserializing file %s: %w", rewardsTreePath, err)
-		}
+			deserializedRewardsFile, err := DeserializeRewardsFile(writeBytes)
+			if err != nil {
+				return fmt.Errorf("Error deserializing file %s: %w", rewardsTreePath, err)
+			}
 
-		// Get the original merkle root
-		downloadedRoot := deserializedRewardsFile.GetHeader().MerkleRoot
+			// Get the original merkle root
+			downloadedRoot := deserializedRewardsFile.GetHeader().MerkleRoot
 
-		// Clear the merkle root so we have a safer comparison after calculating it again
-		deserializedRewardsFile.GetHeader().MerkleRoot = ""
+			// Clear the merkle root so we have a safer comparison after calculating it again
+			deserializedRewardsFile.GetHeader().MerkleRoot = ""
 
-		// Reconstruct the merkle tree from the file data, this should overwrite the stored Merkle Root with a new one
-		deserializedRewardsFile.generateMerkleTree()
+			// Reconstruct the merkle tree from the file data, this should overwrite the stored Merkle Root with a new one
+			deserializedRewardsFile.generateMerkleTree()
 
-		// Get the resulting merkle root
-		calculatedRoot := deserializedRewardsFile.GetHeader().MerkleRoot
+			// Get the resulting merkle root
+			calculatedRoot := deserializedRewardsFile.GetHeader().MerkleRoot
 
-		// Compare the merkle roots to see if the original is correct
-		if !strings.EqualFold(downloadedRoot, calculatedRoot) {
-			return fmt.Errorf("the merkle root from %s does not match the root generated by its tree data (had %s, but generated %s)", url, downloadedRoot, calculatedRoot)
-		}
+			// Compare the merkle roots to see if the original is correct
+			if !strings.EqualFold(downloadedRoot, calculatedRoot) {
+				return fmt.Errorf("the merkle root from %s does not match the root generated by its tree data (had %s, but generated %s)", url, downloadedRoot, calculatedRoot)
+			}
 
-		// Make sure the calculated root matches the canonical one
-		if !strings.EqualFold(calculatedRoot, expectedRoot.Hex()) {
-			return fmt.Errorf("the merkle root from %s does not match the canonical one (had %s, but generated %s)", url, calculatedRoot, expectedRoot.Hex())
-		}
+			// Make sure the calculated root matches the canonical one
+			if !strings.EqualFold(calculatedRoot, expectedRoot.Hex()) {
+				return fmt.Errorf("the merkle root from %s does not match the canonical one (had %s, but generated %s)", url, calculatedRoot, expectedRoot.Hex())
+			}
 
-		// Serialize again so we're sure to have all the correct proofs that we've generated (instead of verifying every proof on the file)
-		localRewardsFile := NewLocalFile[IRewardsFile](
-			deserializedRewardsFile,
-			rewardsTreePath,
-		)
-		err = localRewardsFile.Write()
-		if err != nil {
-			return fmt.Errorf("error saving interval %d file to %s: %w", interval, rewardsTreePath, err)
-		}
+			// Serialize again so we're sure to have all the correct proofs that we've generated (instead of verifying every proof on the file)
+			localRewardsFile := NewLocalFile[IRewardsFile](
+				deserializedRewardsFile,
+				rewardsTreePath,
+			)
+			err = localRewardsFile.Write()
+			if err != nil {
+				return fmt.Errorf("error saving interval %d file to %s: %w", interval, rewardsTreePath, err)
+			}
+
+			return nil
 
-		return nil
+		}
 
+		errBuilder.WriteString(fmt.Sprintf("Downloading files with timeout %v failed.\n", timeout))
 	}
 
 	return fmt.Errorf(errBuilder.String())