From f5939a996c6fcd90254c67f0f165ec8fc2b4edd9 Mon Sep 17 00:00:00 2001 From: zhuxiaolong37 Date: Thu, 22 Feb 2024 16:44:04 +0800 Subject: [PATCH] Add shard upload, shard download verification, and range verification --- oss2/resumable.py | 2 ++ oss2/utils.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/oss2/resumable.py b/oss2/resumable.py index 344d0b85..de558096 100644 --- a/oss2/resumable.py +++ b/oss2/resumable.py @@ -422,6 +422,7 @@ def __download_part(self, part): headers[IF_UNMODIFIED_SINCE] = utils.http_date(self.objectInfo.mtime) result = self.bucket.get_object(self.key, byte_range=(part.start, part.end - 1), headers=headers, params=self.__params) + utils.verify_object_meta(self, part.start, result) utils.copyfileobj_and_verify(result, f, part.end - part.start, request_id=result.request_id) part.part_crc = result.client_crc @@ -610,6 +611,7 @@ def __upload_part(self, part): self._report_progress(self.__finished_size) f.seek(part.start, os.SEEK_SET) + utils.assert_file_not_change(self) headers = _populate_valid_headers(self.__headers, [OSS_REQUEST_PAYER, OSS_TRAFFIC_LIMIT]) if self.__encryption: result = self.bucket.upload_part(self.key, self.__upload_id, part.part_number, diff --git a/oss2/utils.py b/oss2/utils.py index 4263a920..2fcf86cb 100644 --- a/oss2/utils.py +++ b/oss2/utils.py @@ -894,3 +894,19 @@ def to_str(pos): return str(pos) return to_str(start) + '-' + to_str(last) + +def verify_object_meta(self, start, result): + if hasattr(result, 'status') and not hasattr(self.bucket, 'crypto_provider') and result.status == 206: + range_start = result.content_range[result.content_range.find('bytes')+6:result.content_range.find('-')] + if range_start and str(start) != range_start: + raise ValueError("Range get fail, expect offset:{0}, return offset:{1}".format(start, result.content_range)) + if hasattr(result, 'etag') and self.objectInfo.etag != result.etag: + raise ValueError("Range get fail, expect object etag:{0}, return etag:{1}".format(self.objectInfo.etag, result.etag)) + + +def assert_file_not_change(self): + if hasattr(self, 'mtime'): + if self.mtime != os.path.getmtime(self.filename): + raise ValueError("The last modification time of the file has changed, Last file modification time:{0}, The file modification time:{1}".format(self.__mtime, os.path.getmtime(self.filename))) + else: + self.mtime = os.path.getmtime(self.filename)