1, Support object versions lifecycle.

2, Fix resumable download file checkpoint filename with versionid if it specifed.

Author: liyanzhang505

oss2/models.py

@@ -776,27 +776,36 @@ class LifecycleExpiration(object):
     """过期删除操作。
 
     :param days: 表示在文件修改后过了这么多天，就会匹配规则，从而被删除
-    :param date: 表示在该日期之后，规则就一直生效。即每天都会对符合前缀的文件执行删除操作（如，删除），而不管文件是什么时候生成的。
-        *不建议使用*
-    :param created_before_date: delete files if their last modified time earlier than created_before_date
+    :type days: int
 
+    :param date: 表示在该日期之后，规则就一直生效。即每天都会对符合前缀的文件执行删除操作（如，删除），而不管文件是什么时候生成的。*不建议使用*
     :type date: `datetime.date`
+
+    :param created_before_date: delete files if their last modified time earlier than created_before_date
+    :type created_before_date: `datetime.date`
+
+    :param expired_detete_marker: 真实文件删除之后是否自动移除删除标记，适用于多版本场景。
+    :param expired_detete_marker: bool
+
     """
-    def __init__(self, days=None, date=None, created_before_date=None):
+    def __init__(self, days=None, date=None, created_before_date=None, expired_detete_marker=None):
         not_none_fields = 0
         if days is not None:
             not_none_fields += 1
         if date is not None:
             not_none_fields += 1
         if created_before_date is not None:
             not_none_fields += 1
+        if expired_detete_marker is not None:
+            not_none_fields += 1
 
         if not_none_fields > 1:
-            raise ClientError('More than one field(days, date and created_before_date) has been specified')
+            raise ClientError('More than one field(days, date and created_before_date, expired_detete_marker) has been specified')
 
         self.days = days
         self.date = date
         self.created_before_date = created_before_date
+        self.expired_detete_marker = expired_detete_marker
 
 
 class AbortMultipartUpload(object):
@@ -830,18 +839,52 @@ def __init__(self, days=None, created_before_date=None, storage_class=None):
         self.storage_class = storage_class
 
 
+class NoncurrentVersionExpiration(object):
+    """OSS何时将非当前版本的object删除
+
+    :param noncurrent_days: 指定多少天之后删除
+    :type noncurrent_days: int
+    """
+    def __init__(self, noncurrent_days):
+        self.noncurrent_days = noncurrent_days
+
+
+class NoncurrentVersionStorageTransition(object):
+    """生命周期内，OSS何时将指定Object的非当前版本转储为IA或者Archive存储类型。
+
+    :param noncurrent_days: 多少天之后转存储
+    :type noncurrent_days: int
+    """
+    def __init__(self, noncurrent_days, storage_class):
+        self.noncurrent_days = noncurrent_days
+        self.storage_class = storage_class
+
+
 class LifecycleRule(object):
     """生命周期规则。
 
     :param id: 规则名
+    :type id: str
+
     :param prefix: 只有文件名匹配该前缀的文件才适用本规则
+    :type prefix: str
+
     :param expiration: 过期删除操作。
     :type expiration: :class:`LifecycleExpiration`
+
     :param status: 启用还是禁止该规则。可选值为 `LifecycleRule.ENABLED` 或 `LifecycleRule.DISABLED`
+
     :param storage_transitions: 存储类型转换规则
-    :type storage_transitions: :class:`StorageTransition`
+    :type storage_transitions: list of class:`StorageTransition <oss2.models.StorageTransition>`
+
     :param tagging: object tagging 规则
-    :type tagging: :class:`Tagging`
+    :type tagging: :class:`Tagging <oss2.models.StorageTransition>`
+    
+    :param noncurrent_version_expiration: 指定Object非当前版本生命周期规则的过期属性。适用于多版本场景。
+    :type noncurrent_version_expiration class:`NoncurrentVersionExpiration <oss2.models.NoncurrentVersionExpiration>`
+
+    :param noncurrent_version_sotrage_transitions: 在有效生命周期中，OSS何时将指定Object的非当前版本转储为IA或者Archive存储类型，适用于多版本场景。
+    :type noncurrent_version_sotrage_transitions: list of class:`NoncurrentVersionStorageTransition <oss2.models.NoncurrentVersionStorageTransition>`
     """
 
     ENABLED = 'Enabled'
@@ -850,21 +893,25 @@ class LifecycleRule(object):
     def __init__(self, id, prefix,
                  status=ENABLED, expiration=None,
                  abort_multipart_upload=None,
-                 storage_transitions=None, tagging=None):
+                 storage_transitions=None, tagging=None,
+                 noncurrent_version_expiration=None,
+                 noncurrent_version_sotrage_transitions=None):
         self.id = id
         self.prefix = prefix
         self.status = status
         self.expiration = expiration
         self.abort_multipart_upload = abort_multipart_upload
         self.storage_transitions = storage_transitions
         self.tagging = tagging
+        self.noncurrent_version_expiration = noncurrent_version_expiration
+        self.noncurrent_version_sotrage_transitions = noncurrent_version_sotrage_transitions
 
 
 class BucketLifecycle(object):
     """Bucket的生命周期配置。
 
     :param rules: 规则列表，
-    :type rules: list of :class:`LifecycleRule`
+    :type rules: list of :class:`LifecycleRule <oss2.models.LifecycleRule>`
     """
     def __init__(self, rules=None):
         self.rules = rules or []

oss2/resumable.py

@@ -240,7 +240,7 @@ def _populate_valid_headers(headers=None, valid_keys=None):
 
 class _ResumableOperation(object):
     def __init__(self, bucket, key, filename, size, store,
-                 progress_callback=None):
+                 progress_callback=None, versionid=None):
         self.bucket = bucket
         self.key = to_string(key)
         self.filename = filename
@@ -249,7 +249,12 @@ def __init__(self, bucket, key, filename, size, store,
         self._abspath = os.path.abspath(filename)
 
         self.__store = store
-        self.__record_key = self.__store.make_store_key(bucket.bucket_name, self.key, self._abspath)
+
+        if versionid is None:
+            self.__record_key = self.__store.make_store_key(bucket.bucket_name, self.key, self._abspath)
+        else:
+            self.__record_key = self.__store.make_store_key(bucket.bucket_name, self.key, self._abspath, versionid)
+
         logger.debug("Init _ResumableOperation, record_key: {0}".format(self.__record_key))
 
         # protect self.__progress_callback
@@ -295,9 +300,13 @@ def __init__(self, bucket, key, filename, objectInfo,
                  num_threads=None,
                  params=None,
                  headers=None):
+        versionid = None
+        if params is not None and params.get('versionId') is not None:
+            versionid = params.get('versionId')
         super(_ResumableDownloader, self).__init__(bucket, key, filename, objectInfo.size,
                                                    store or ResumableDownloadStore(),
-                                                   progress_callback=progress_callback)
+                                                   progress_callback=progress_callback, 
+                                                   versionid=versionid)
         self.objectInfo = objectInfo
 
         self.__part_size = defaults.get(part_size, defaults.multiget_part_size)
@@ -718,10 +727,15 @@ def __init__(self, root=None, dir=None):
         super(ResumableDownloadStore, self).__init__(root or os.path.expanduser('~'), dir or _DOWNLOAD_TEMP_DIR)
 
     @staticmethod
-    def make_store_key(bucket_name, key, filename):
+    def make_store_key(bucket_name, key, filename, versionid=None):
         filepath = _normalize_path(filename)
+        oss_pathname = None
 
-        oss_pathname = 'oss://{0}/{1}'.format(bucket_name, key)
+        if versionid is None:
+            oss_pathname = 'oss://{0}/{1}'.format(bucket_name, key)
+        else:
+            oss_pathname = 'oss://{0}/{1}?versionid={2}'.format(bucket_name, key, versionid)
+        
         return utils.md5_string(oss_pathname) + '-' + utils.md5_string(filepath) + '-download'
 
 

oss2/xml_utils.py

@@ -47,7 +47,9 @@
                      REDIRECT_TYPE_MIRROR,
                      REDIRECT_TYPE_EXTERNAL,
                      REDIRECT_TYPE_INTERNAL,
-                     REDIRECT_TYPE_ALICDN)
+                     REDIRECT_TYPE_ALICDN,
+                     NoncurrentVersionStorageTransition,
+                     NoncurrentVersionExpiration)
 
 from .select_params import (SelectJsonTypes, SelectParameters)
 
@@ -583,6 +585,10 @@ def parse_lifecycle_expiration(expiration_node):
         expiration.days = _find_int(expiration_node, 'Days')
     elif expiration_node.find('Date') is not None:
         expiration.date = iso8601_to_date(_find_tag(expiration_node, 'Date'))
+    elif expiration_node.find('CreatedBeforeDate') is not None:
+        expiration.created_before_date = iso8601_to_date(_find_tag(expiration_node, 'CreatedBeforeDate'))
+    elif expiration_node.find('ExpiredObjectDeleteMarker') is not None:
+        expiration.expired_detete_marker = _find_bool(expiration_node, 'ExpiredObjectDeleteMarker')
 
     return expiration
 
@@ -629,6 +635,25 @@ def parse_lifecycle_object_taggings(lifecycle_tagging_nodes):
 
     return Tagging(tagging_rule)
 
+def parse_lifecycle_version_expiration(version_expiration_node):
+    if version_expiration_node is None:
+        return None
+
+    noncurrent_days = _find_int(version_expiration_node, 'NoncurrentDays')
+    expiration = NoncurrentVersionExpiration(noncurrent_days)
+
+    return expiration
+
+def parse_lifecycle_verison_storage_transitions(version_storage_transition_nodes):
+    version_storage_transitions = []
+    for transition_node in version_storage_transition_nodes:
+        storage_class = _find_tag(transition_node, 'StorageClass')
+        non_crurrent_days = _find_int(transition_node, 'NoncurrentDays')
+        version_storage_transition = NoncurrentVersionStorageTransition(non_crurrent_days, storage_class)
+        version_storage_transitions.append(version_storage_transition)
+
+    return version_storage_transitions
+
 def parse_get_bucket_lifecycle(result, body):
 
     root = ElementTree.fromstring(body)
@@ -639,14 +664,19 @@ def parse_get_bucket_lifecycle(result, body):
         abort_multipart_upload = parse_lifecycle_abort_multipart_upload(rule_node.find('AbortMultipartUpload'))
         storage_transitions = parse_lifecycle_storage_transitions(rule_node.findall('Transition'))
         tagging = parse_lifecycle_object_taggings(rule_node.findall('Tag'))
+        noncurrent_version_expiration = parse_lifecycle_version_expiration(rule_node.find('NoncurrentVersionExpiration'))
+        noncurrent_version_sotrage_transitions = parse_lifecycle_verison_storage_transitions(rule_node.findall('NoncurrentVersionTransition'))
+
         rule = LifecycleRule(
             _find_tag(rule_node, 'ID'),
             _find_tag(rule_node, 'Prefix'),
             status=_find_tag(rule_node, 'Status'),
             expiration=expiration,
             abort_multipart_upload=abort_multipart_upload,
             storage_transitions=storage_transitions,
-            tagging=tagging 
+            tagging=tagging,
+            noncurrent_version_expiration = noncurrent_version_expiration,
+            noncurrent_version_sotrage_transitions = noncurrent_version_sotrage_transitions
             )
         result.rules.append(rule)
 
@@ -852,6 +882,8 @@ def to_put_bucket_lifecycle(bucket_lifecycle):
                 _add_text_child(expiration_node, 'Date', date_to_iso8601(expiration.date))
             elif expiration.created_before_date is not None:
                 _add_text_child(expiration_node, 'CreatedBeforeDate', date_to_iso8601(expiration.created_before_date))
+            elif expiration.expired_detete_marker is not None:
+                _add_text_child(expiration_node, 'ExpiredObjectDeleteMarker', str(expiration.expired_detete_marker))
 
         abort_multipart_upload = rule.abort_multipart_upload
         if abort_multipart_upload:
@@ -880,6 +912,19 @@ def to_put_bucket_lifecycle(bucket_lifecycle):
                 tag_node = ElementTree.SubElement(rule_node, 'Tag')
                 _add_text_child(tag_node, 'Key', key)
                 _add_text_child(tag_node, 'Value', tagging_rule[key])
+
+        noncurrent_version_expiration = rule.noncurrent_version_expiration
+        if noncurrent_version_expiration is not None:
+            version_expiration_node = ElementTree.SubElement(rule_node, 'NoncurrentVersionExpiration')
+            _add_text_child(version_expiration_node, 'NoncurrentDays', str(noncurrent_version_expiration.noncurrent_days))
+
+        noncurrent_version_sotrage_transitions = rule.noncurrent_version_sotrage_transitions
+        if noncurrent_version_sotrage_transitions is not None:
+            for noncurrent_version_sotrage_transition in noncurrent_version_sotrage_transitions:
+                version_transition_node = ElementTree.SubElement(rule_node, 'NoncurrentVersionTransition')
+                _add_text_child(version_transition_node, 'NoncurrentDays', str(noncurrent_version_sotrage_transition.noncurrent_days))
+                _add_text_child(version_transition_node, 'StorageClass', str(noncurrent_version_sotrage_transition.storage_class))
+
     return _node_to_string(root)