Adding retries for failed copy_blob() calls in system tests.

Fixes #2289.

Author: dhermes

system_tests/storage.py

@@ -31,9 +31,19 @@
 from retry import RetryResult
 
 
-retry_429 = RetryErrors(exceptions.TooManyRequests)
 HTTP = httplib2.Http()
-_helpers.PROJECT = TESTS_PROJECT
+
+
+def _bad_copy(bad_request):
+    """Predicate: pass only exceptions for a failed copyTo."""
+    err_msg = bad_request.message
+    return (err_msg.startswith('No file found in request. (POST') and
+            'copyTo' in err_msg)
+
+
+retry_429 = RetryErrors(exceptions.TooManyRequests)
+retry_bad_copy = RetryErrors(exceptions.BadRequest,
+                             error_predicate=_bad_copy)
 
 
 class Config(object):
@@ -47,6 +57,7 @@ class Config(object):
 
 
 def setUpModule():
+    _helpers.PROJECT = TESTS_PROJECT
     Config.CLIENT = storage.Client()
     bucket_name = 'new' + unique_resource_id()
     # In the **very** rare case the bucket name is reserved, this
@@ -191,7 +202,8 @@ def test_copy_existing_file(self):
         blob.upload_from_filename(filename)
         self.case_blobs_to_delete.append(blob)
 
-        new_blob = self.bucket.copy_blob(blob, self.bucket, 'CloudLogoCopy')
+        new_blob = retry_bad_copy(self.bucket.copy_blob)(
+            blob, self.bucket, 'CloudLogoCopy')
         self.case_blobs_to_delete.append(new_blob)
 
         base_contents = blob.download_as_string()
@@ -217,7 +229,8 @@ def setUpClass(cls):
 
         # Copy main blob onto remaining in FILENAMES.
         for filename in cls.FILENAMES[1:]:
-            new_blob = cls.bucket.copy_blob(blob, cls.bucket, filename)
+            new_blob = retry_bad_copy(cls.bucket.copy_blob)(
+                blob, cls.bucket, filename)
             cls.suite_blobs_to_delete.append(new_blob)
 
     @classmethod
@@ -278,7 +291,8 @@ def setUpClass(cls):
         blob.upload_from_filename(simple_path)
         cls.suite_blobs_to_delete = [blob]
         for filename in cls.FILENAMES[1:]:
-            new_blob = cls.bucket.copy_blob(blob, cls.bucket, filename)
+            new_blob = retry_bad_copy(cls.bucket.copy_blob)(
+                blob, cls.bucket, filename)
             cls.suite_blobs_to_delete.append(new_blob)
 
     @classmethod