in lambda, fetch_listing with retry_on_notfound, fixes #424 (#442)

* in lambda, fetch_listing with retry_on_notfound, fixes #424

* better tests for fetch_listing

jobs/buildhub/inventory_to_records.py

@@ -107,9 +107,13 @@ async def fetch_json(
         raise
 
 
-async def fetch_listing(session, url):
+async def fetch_listing(session, url, retry_on_notfound=False):
     try:
-        data = await fetch_json(session, url)
+        data = await fetch_json(
+            session,
+            url,
+            retry_on_notfound=retry_on_notfound,
+        )
         return data['prefixes'], data['files']
     except (aiohttp.ClientError, KeyError, ValueError) as e:
         raise ValueError("Could not fetch '{}': {}".format(url, e))

jobs/buildhub/lambda_s3_event.py

@@ -130,7 +130,11 @@ async def main(loop, event):
                     # For multi we just check the associated archive
                     # is here already.
                     parent_folder = re.sub('multi/.+$', 'multi/', url)
-                    _, files = await fetch_listing(session, parent_folder)
+                    _, files = await fetch_listing(
+                        session,
+                        parent_folder,
+                        retry_on_notfound=True
+                    )
                     for f in files:
                         rc_url = parent_folder + f['name']
                         if utils.is_build_url(product, rc_url):

jobs/tests/test_inventory_to_records.py

@@ -228,6 +228,29 @@ async def test_raises_valueerror_if_html(self):
                     self.url
                 )
 
+    async def test_jsonnotfound_exception(self):
+        with aioresponses() as m:
+            m.get(self.url, status=404)
+            with self.assertRaises(inventory_to_records.JSONFileNotFound):
+                await inventory_to_records.fetch_listing(
+                    self.session,
+                    self.url,
+                )
+
+    async def test_retries_in_case_of_404_response(self):
+        with aioresponses() as m:
+            m.get(self.url, status=404)
+            m.get(self.url, payload={
+                'prefixes': ['a/', 'b/'],
+                'files': [{'name': 'foo.txt'}]
+            })
+            received = await inventory_to_records.fetch_listing(
+                self.session,
+                self.url,
+                retry_on_notfound=True
+            )
+            assert received == (['a/', 'b/'], [{'name': 'foo.txt'}])
+
 
 class FetchNightlyMetadata(asynctest.TestCase):
     async def setUp(self):