Merge pull request #162 from islue/s3_multipart

MultiPartUpload.get_all_parts return 1,000 multipart uploads max

Author: timvaillancourt

conf/mongodb-consistent-backup.example.conf

@@ -55,6 +55,7 @@ production:
   #    bucket_name: [AWS S3 Bucket Name]
   #    bucket_prefix: [prefix] (default: /)
   #    threads: [1+]           (default: 1 per CPU)
+  #    chunk_size_mb: [1+]     (default: 50)
   #    secure: [true|false]    (default: true)
   #    retries: [1+]           (default: 5)
   #    acl: [acl_str]          (default: none)

mongodb_consistent_backup/Upload/S3/S3.py

@@ -1,6 +1,7 @@
 import os
 import logging
 
+import boto.s3.multipart
 from copy_reg import pickle
 from math import ceil
 from multiprocessing import Pool
@@ -102,7 +103,10 @@ def run(self):
                 self._pool.close()
                 self._pool.join()
 
-                if len(self._multipart.get_all_parts()) == chunk_count:
+                part_count = 0
+                for part in boto.s3.multipart.part_lister(self._multipart):
+                  part_count += 1
+                if part_count == chunk_count:
                     self._multipart.complete_upload()
                     key = self.bucket.get_key(key_name)
                     if self.s3_acl: