adding exponential backoff to aws products api (#20)

* adding exponential backoff to aws products api
Signed-off-by: vsoch <vsoch@users.noreply.github.com>

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/converged-computing/cloud-select/tree/main) (0.0.x)
+ - exponential backoff added for using aws products/prices api (0.0.15)
  - dbshell command added with tutorial, testing for queries and shell (0.0.14)
  - bugfix to allow oras without auth for pull (0.0.13)
    - addition of basic container tutorial

cloud_select/main/cloud/aws/client.py

@@ -4,7 +4,9 @@
 # SPDX-License-Identifier: (MIT)
 
 import json
+import random
 import re
+import time
 
 import cloud_select.utils as utils
 from cloud_select.logger import logger
@@ -27,6 +29,10 @@ def __init__(self, **kwargs):
         self.ec2_client = None
         self.pricing_cli = None
 
+        # Exponential backoff for prices API
+        self.min_sleep_time = kwargs.get("min_sleep_time") or 1e-2
+        self.max_retries = kwargs.get("max_retries") or 15
+
         # This currently has two pieces - billing and instances (different APIs)
         self._set_services(kwargs.get("cache_only", False))
         super(AmazonCloud, self).__init__()
@@ -38,19 +44,33 @@ def prices(self):
         if not self.has_pricing_auth:
             return self.fail_message("prices, authentication not set.")
 
+        from botocore.exceptions import ClientError
+
         # Get services first - there are almost 2k! Look for compute engine
         logger.info(f"Retrieving prices for {self.name}.")
 
         # Keep a price that matches any region we care about
         regex = "(%s)" % "|".join(self.regions)
         logger.debug(f"Searching for region regex {regex}")
 
+        # Keep track of how many times we've tried
+        retries = 0
         next_token = ""
         prices = []
         while True:
-            response = self.pricing_cli.get_products(
-                ServiceCode="AmazonEC2", NextToken=next_token
-            )
+            try:
+                response = self.pricing_cli.get_products(
+                    ServiceCode="AmazonEC2", NextToken=next_token
+                )
+            # Be generous and retry for any client error
+            except ClientError as err:
+                if "Rate exceeded" not in err.args[0] or retries > self.max_retries:
+                    raise
+                retries += 1
+                sleep = self.min_sleep_time * random.randint(1, 2**retries)
+                time.sleep(sleep)
+                continue
+
             if not response.get("NextToken"):
                 break
             next_token = response.get("NextToken")

cloud_select/version.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (MIT)
 
-__version__ = "0.0.14"
+__version__ = "0.0.15"
 AUTHOR = "Vanessa Sochat"
 EMAIL = "vsoch@users.noreply.github.com"
 NAME = "cloud-select-tool"