Merge pull request #187 from scality/bugfix/ZENKO-723_Explicit_MPU

bugfix(zenko-723): Explicitly define when MPUs happen

tests/docker-entrypoint.sh

@@ -22,4 +22,4 @@ fi
 python create_buckets.py
 
 # Disable cache to run in a read-only container
-exec pytest -s -p no:cacheprovider -ra "${PYTEST_XDIST_ARGS}" "$@"
+exec pytest -s -p no:cacheprovider -raR "${PYTEST_XDIST_ARGS}" "$@"

tests/requirements.txt

@@ -152,6 +152,9 @@ pytest-dotenv==0.1.0 \
 pytest-forked==0.2 \
     --hash=sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805 \
     --hash=sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08
+pytest-rerunfailures==4.1 \
+    --hash=sha256:be6bf93ed618c8899aeb6721c24f8009c769879a3b4931e05650f3c173ec17c5 \
+    --hash=sha256:deeecaa0765038107eccbdeacaacdff186331d4958f417d132c490735398ca22
 pytest-runner==4.2 \
     --hash=sha256:d23f117be39919f00dd91bffeb4f15e031ec797501b717a245e377aee0f577be \
     --hash=sha256:d987fec1e31287592ffe1cb823a8c613c533db4c6aaca0ee1191dbc91e2fcc61

tests/zenko_e2e/cloudserver/test_mpu.py

@@ -1,27 +1,22 @@
 import pytest
 from ..fixtures import *
+from .. import util
 
 
 @pytest.mark.conformance
 def test_mpu_aws(aws_ep_bucket, aws_target_bucket, mpufile, objkey):
     util.mark_test('AWS MPU UPLOAD')
-    aws_ep_bucket.create('zenko-aws-mpu-bucket')
-    aws_ep_bucket.put_object(
-        Body=mpufile,
-        Key=objkey
-    )
+    aws_ep_bucket.create()
+    util.upload_object(aws_ep_bucket, objkey, mpufile)
     util.remark('Done uploading file, downloading and checking hash')
     assert util.check_object(objkey, mpufile, aws_ep_bucket, aws_target_bucket)
 
 
 @pytest.mark.conformance
 def test_mpu_gcp(gcp_ep_bucket, gcp_target_bucket, mpufile, objkey):
     util.mark_test('GCP MPU UPLOAD')
-    gcp_ep_bucket.create('zenko-gcp-mpu-bucket')
-    gcp_ep_bucket.put_object(
-        Body=mpufile,
-        Key=objkey
-    )
+    gcp_ep_bucket.create()
+    util.upload_object(gcp_ep_bucket, objkey, mpufile)
     util.remark('Done uploading file, downloading and checking hash')
     assert util.check_object(objkey, mpufile, gcp_ep_bucket, gcp_target_bucket)
 
@@ -31,11 +26,8 @@ def test_mpu_gcp(gcp_ep_bucket, gcp_target_bucket, mpufile, objkey):
 @pytest.mark.conformance
 def test_mpu_azure(azure_ep_bucket, azure_target_bucket, mpufile, objkey):
     util.mark_test('AZURE MPU UPLOAD')
-    azure_ep_bucket.create('zenko-azure-mpu-bucket')
-    azure_ep_bucket.put_object(
-        Body=mpufile,
-        Key=objkey
-    )
+    azure_ep_bucket.create()
+    util.upload_object(azure_ep_bucket, objkey, mpufile)
     util.remark('Done uploading file, downloading and checking hash')
     assert util.check_object(
         objkey, mpufile, azure_ep_bucket, azure_target_bucket)
@@ -46,10 +38,7 @@ def test_mpu_azure(azure_ep_bucket, azure_target_bucket, mpufile, objkey):
 def test_mpu_wasabi(wasabi_ep_bucket, wasabi_target_bucket, mpufile, objkey):
     util.mark_test('WASABI MPU UPLOAD')
     wasabi_ep_bucket.create()
-    wasabi_ep_bucket.put_object(
-        Body=mpufile,
-        Key=objkey
-    )
+    util.upload_object(wasabi_ep_bucket, objkey, mpufile)
     util.remark('Done uploading file, downloading and checking hash')
     assert util.check_object(
         objkey, mpufile, wasabi_ep_bucket, wasabi_target_bucket)
@@ -62,10 +51,7 @@ def test_mpu_digital_ocean(digital_ocean_ep_bucket,
                            digital_ocean_target_bucket, mpufile, objkey):
     util.mark_test('DIGITAL OCEAN MPU UPLOAD')
     digital_ocean_ep_bucket.create()
-    digital_ocean_ep_bucket.put_object(
-        Body=mpufile,
-        Key=objkey
-    )
+    util.upload_object(digital_ocean_ep_bucket, objkey, mpufile)
     util.remark('Done uploading file, downloading and checking hash')
     assert util.check_object(
         objkey, mpufile, digital_ocean_ep_bucket, digital_ocean_target_bucket)

tests/zenko_e2e/cloudserver/test_replication.py

@@ -15,6 +15,7 @@
 def test_aws_1_1(aws_crr_bucket, aws_crr_target_bucket, objkey, datafile):
     util.mark_test('AWS 1-1 REPLICATION')
     data = datafile()
+    util.upload_object(aws_crr_bucket, objkey, data)
     aws_crr_bucket.put_object(
         Body=data,
         Key=objkey
@@ -28,24 +29,19 @@ def test_aws_1_1(aws_crr_bucket, aws_crr_target_bucket, objkey, datafile):
 def test_gcp_1_1(gcp_crr_bucket, gcp_crr_target_bucket, objkey, datafile):
     util.mark_test('GCP 1-1 REPLICATION')
     data = datafile()
-    gcp_crr_bucket.put_object(
-        Body=data,
-        Key=objkey
-    )
+    util.upload_object(gcp_crr_bucket, objkey, data)
     assert util.check_object(
         objkey, data, gcp_crr_bucket, gcp_crr_target_bucket, timeout=30)
 
 
+@pytest.mark.flaky(reruns=3)
 @pytest.mark.parametrize('datafile', [testfile, mpufile])
 @pytest.mark.conformance
 def test_azure_1_1(
         azure_crr_bucket, azure_crr_target_bucket, objkey, datafile):
     util.mark_test('AZURE 1-1 REPLICATION')
     data = datafile()
-    azure_crr_bucket.put_object(
-        Body=data,
-        Key=objkey
-    )
+    util.upload_object(azure_crr_bucket, objkey, data)
     assert util.check_object(
         objkey,
         data,
@@ -55,14 +51,13 @@ def test_azure_1_1(
 
 
 @pytest.mark.skip(reason='Wasabi not implemented in CI')
+@pytest.mark.parametrize('datafile', [testfile, mpufile])
 @pytest.mark.conformance
 def test_wasabi_1_1(wasabi_crr_bucket,
-                    wasabi_crr_target_bucket, testfile, objkey):
+                    wasabi_crr_target_bucket, objkey, datafile):
     util.mark_test('AZURE 1-1 REPLICATION')
-    wasabi_crr_bucket.put_object(
-        Body=testfile,
-        Key=objkey
-    )
+    data = datafile()
+    util.upload_object(wasabi_crr_bucket, objkey, data)
     assert util.check_object(
         objkey,
         testfile,
@@ -72,19 +67,18 @@ def test_wasabi_1_1(wasabi_crr_bucket,
 
 
 @pytest.mark.skip(reason='Not implemented in CI')
+@pytest.mark.parametrize('datafile', [testfile, mpufile])
 @pytest.mark.conformance
 def test_multi_1_M(  # pylint: disable=invalid-name, too-many-arguments
         multi_crr_bucket,
         aws_crr_target_bucket,
         gcp_crr_target_bucket,
         azure_crr_target_bucket,
-        testfile,
-        objkey):
+        objkey,
+        datafile):
     util.mark_test("MULTI 1-M REPLICATION")
-    multi_crr_bucket.put_object(
-        Body=testfile,
-        Key=objkey
-    )
+    data = datafile()
+    util.upload_object(multi_crr_bucket, objkey, data)
     assert util.check_object(objkey, testfile,
                              multi_crr_bucket,
                              aws_crr_target_bucket,

tests/zenko_e2e/util/bucket.py

@@ -3,6 +3,7 @@
 import hashlib
 import tempfile
 import time
+from boto3.s3.transfer import TransferConfig
 import zenko_e2e.conf as conf
 
 _log = logging.getLogger('util.bucket')  # pylint: disable=invalid-name
@@ -127,3 +128,16 @@ def cleanup_bucket(bucket, replicated=False, delete_bucket=True): # noqa pylint:
             key_marker = version_list['NextKeyMarker']
     if delete_bucket:
         bucket.delete()
+
+
+UPLOAD_CONFIG = TransferConfig(multipart_threshold=10 * 1024 * 1024)  # 10MB
+
+
+def upload_object(bucket, key, data, **kwargs):
+    with tempfile.TemporaryFile() as upload:
+        upload.write(data)
+        upload.seek(0)
+        return bucket.upload_fileobj(upload,
+                                     key,
+                                     Config=UPLOAD_CONFIG,
+                                     ExtraArgs=kwargs)