Skip to content

Commit 6345224

Browse files
author
Rakshil Modi
committed
Adding no-overwrite flag for S3 upload
Removed unwanted comments for test
1 parent 751cab8 commit 6345224

File tree

6 files changed

+265
-2
lines changed

6 files changed

+265
-2
lines changed

‎awscli/customizations/s3/results.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@
2121

2222
from awscli.compat import ensure_text_type, queue
2323
from awscli.customizations.s3.subscribers import OnDoneFilteredSubscriber
24-
from awscli.customizations.s3.utils import WarningResult, human_readable_size
24+
from awscli.customizations.s3.utils import (
25+
WarningResult,
26+
create_warning,
27+
human_readable_size,
28+
)
2529
from awscli.customizations.utils import uni_print
2630

2731
LOGGER = logging.getLogger(__name__)
@@ -123,6 +127,12 @@ def _on_failure(self, future, e):
123127
if isinstance(e, FatalError):
124128
error_result_cls = ErrorResult
125129
self._result_queue.put(error_result_cls(exception=e))
130+
elif self._is_precondition_failed(e):
131+
LOGGER.debug(
132+
"Warning: Skipping file %s as it already exists on %s",
133+
self._src,
134+
self._dest,
135+
)
126136
else:
127137
self._result_queue.put(
128138
FailureResult(
@@ -133,6 +143,14 @@ def _on_failure(self, future, e):
133143
)
134144
)
135145

146+
def _is_precondition_failed(self, exception):
147+
"""Check if this is a PreconditionFailed error"""
148+
return (
149+
hasattr(exception, 'response')
150+
and exception.response.get('Error', {}).get('Code')
151+
== 'PreconditionFailed'
152+
)
153+
136154

137155
class BaseResultHandler:
138156
"""Base handler class to be called in the ResultProcessor"""

‎awscli/customizations/s3/subcommands.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,15 @@
642642
),
643643
}
644644

645+
NO_OVERWRITE = {
646+
'name': 'no-overwrite',
647+
'action': 'store_true',
648+
'help_text': (
649+
"This flag prevents overwriting of files at the destination. With this flag, "
650+
"only files not present at the destination will be transferred."
651+
),
652+
}
653+
645654
TRANSFER_ARGS = [
646655
DRYRUN,
647656
QUIET,
@@ -1057,7 +1066,14 @@ class CpCommand(S3TransferCommand):
10571066
}
10581067
]
10591068
+ TRANSFER_ARGS
1060-
+ [METADATA, COPY_PROPS, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE]
1069+
+ [
1070+
METADATA,
1071+
COPY_PROPS,
1072+
METADATA_DIRECTIVE,
1073+
EXPECTED_SIZE,
1074+
RECURSIVE,
1075+
NO_OVERWRITE,
1076+
]
10611077
)
10621078

10631079

@@ -1081,6 +1097,7 @@ class MvCommand(S3TransferCommand):
10811097
METADATA_DIRECTIVE,
10821098
RECURSIVE,
10831099
VALIDATE_SAME_S3_PATHS,
1100+
NO_OVERWRITE,
10841101
]
10851102
)
10861103

‎awscli/customizations/s3/utils.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,7 @@ def map_put_object_params(cls, request_params, cli_params):
489489
cls._set_sse_c_request_params(request_params, cli_params)
490490
cls._set_request_payer_param(request_params, cli_params)
491491
cls._set_checksum_algorithm_param(request_params, cli_params)
492+
cls._set_no_overwrite_param(request_params, cli_params)
492493

493494
@classmethod
494495
def map_get_object_params(cls, request_params, cli_params):
@@ -558,6 +559,12 @@ def map_delete_object_params(cls, request_params, cli_params):
558559
def map_list_objects_v2_params(cls, request_params, cli_params):
559560
cls._set_request_payer_param(request_params, cli_params)
560561

562+
@classmethod
563+
def _set_no_overwrite_param(cls, request_params, cli_params):
564+
"""Map No overwrite header with IfNoneMatch"""
565+
if cli_params.get('no_overwrite'):
566+
request_params['IfNoneMatch'] = "*"
567+
561568
@classmethod
562569
def _set_request_payer_param(cls, request_params, cli_params):
563570
if cli_params.get('request_payer'):

‎tests/functional/s3/test_cp_command.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,107 @@ def test_operations_used_in_recursive_download(self):
296296
)
297297
self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
298298

299+
def test_no_overwrite_flag_when_object_not_exists_on_target(self):
300+
full_path = self.files.create_file('foo.txt', 'mycontent')
301+
cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
302+
self.parsed_responses = [
303+
{'ETag': '"c8afdb36c52cf4727836669019e69222"'}
304+
]
305+
self.run_cmd(cmdline, expected_rc=0)
306+
# Verify putObject was called
307+
self.assertEqual(len(self.operations_called), 1)
308+
self.assertEqual(self.operations_called[0][0].name, 'PutObject')
309+
# Verify the IfNoneMatch condition was set in the request
310+
self.assertEqual(self.operations_called[0][1]['IfNoneMatch'], '*')
311+
312+
def test_no_overwrite_flag_when_object_exists_on_target(self):
313+
full_path = self.files.create_file('foo.txt', 'mycontent')
314+
cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
315+
# Set up the response to simulate a PreconditionFailed error
316+
self.http_response.status_code = 412
317+
self.parsed_responses = [
318+
{
319+
'Error': {
320+
'Code': 'PreconditionFailed',
321+
'Message': 'At least one of the pre-conditions you specified did not hold',
322+
'Condition': 'If-None-Match',
323+
}
324+
}
325+
]
326+
self.run_cmd(cmdline, expected_rc=0)
327+
# Verify PutObject was attempted with IfNoneMatch
328+
self.assertEqual(len(self.operations_called), 1)
329+
self.assertEqual(self.operations_called[0][0].name, 'PutObject')
330+
self.assertEqual(self.operations_called[0][1]['IfNoneMatch'], '*')
331+
332+
def test_no_overwrite_flag_multipart_upload_when_object_not_exists_on_target(
333+
self,
334+
):
335+
# Create a large file that will trigger multipart upload
336+
full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024**2))
337+
cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
338+
339+
# Set up responses for multipart upload
340+
self.parsed_responses = [
341+
{'UploadId': 'foo'}, # CreateMultipartUpload response
342+
{'ETag': '"foo-1"'}, # UploadPart response
343+
{'ETag': '"foo-2"'}, # UploadPart response
344+
{}, # CompleteMultipartUpload response
345+
]
346+
self.run_cmd(cmdline, expected_rc=0)
347+
# Verify all multipart operations were called
348+
self.assertEqual(len(self.operations_called), 4)
349+
self.assertEqual(
350+
self.operations_called[0][0].name, 'CreateMultipartUpload'
351+
)
352+
self.assertEqual(self.operations_called[1][0].name, 'UploadPart')
353+
self.assertEqual(self.operations_called[2][0].name, 'UploadPart')
354+
self.assertEqual(
355+
self.operations_called[3][0].name, 'CompleteMultipartUpload'
356+
)
357+
# Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
358+
self.assertEqual(self.operations_called[3][1]['IfNoneMatch'], '*')
359+
360+
def test_no_overwrite_flag_multipart_upload_when_object_exists_on_target(
361+
self,
362+
):
363+
# Create a large file that will trigger multipart upload
364+
full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024**2))
365+
cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
366+
# Set up responses for multipart upload
367+
self.parsed_responses = [
368+
{'UploadId': 'foo'}, # CreateMultipartUpload response
369+
{'ETag': '"foo-1"'}, # UploadPart response
370+
{'ETag': '"foo-2"'}, # UploadPart response
371+
{
372+
'Error': {
373+
'Code': 'PreconditionFailed',
374+
'Message': 'At least one of the pre-conditions you specified did not hold',
375+
'Condition': 'If-None-Match',
376+
}
377+
}, # PreconditionFailed error for CompleteMultipart Upload
378+
{}, # AbortMultipartUpload response
379+
]
380+
# Checking for success as file is skipped
381+
self.run_cmd(cmdline, expected_rc=0)
382+
# Set up the response to simulate a PreconditionFailed error
383+
self.http_response.status_code = 412
384+
# Verify all multipart operations were called
385+
self.assertEqual(len(self.operations_called), 5)
386+
self.assertEqual(
387+
self.operations_called[0][0].name, 'CreateMultipartUpload'
388+
)
389+
self.assertEqual(self.operations_called[1][0].name, 'UploadPart')
390+
self.assertEqual(self.operations_called[2][0].name, 'UploadPart')
391+
self.assertEqual(
392+
self.operations_called[3][0].name, 'CompleteMultipartUpload'
393+
)
394+
self.assertEqual(
395+
self.operations_called[4][0].name, 'AbortMultipartUpload'
396+
)
397+
# Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
398+
self.assertEqual(self.operations_called[3][1]['IfNoneMatch'], '*')
399+
299400
def test_dryrun_download(self):
300401
self.parsed_responses = [self.head_object_response()]
301402
target = self.files.full_path('file.txt')

‎tests/functional/s3/test_mv_command.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,112 @@ def test_download_with_checksum_mode_crc32(self):
316316
self.operations_called[1][1]['ChecksumMode'], 'ENABLED'
317317
)
318318

319+
def test_mv_no_overwrite_flag_when_object_not_exists_on_target(self):
320+
full_path = self.files.create_file('foo.txt', 'contents')
321+
cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
322+
self.run_cmd(cmdline, expected_rc=0)
323+
# Verify putObject was called
324+
self.assertEqual(len(self.operations_called), 1)
325+
self.assertEqual(self.operations_called[0][0].name, 'PutObject')
326+
# Verify the IfNoneMatch condition was set in the request
327+
self.assertEqual(self.operations_called[0][1]['IfNoneMatch'], '*')
328+
# Verify source file was deleted (move operation)
329+
self.assertFalse(os.path.exists(full_path))
330+
331+
def test_mv_no_overwrite_flag_when_object_exists_on_target(self):
332+
full_path = self.files.create_file('foo.txt', 'mycontent')
333+
cmdline = (
334+
f'{self.prefix} {full_path} s3://bucket/foo.txt --no-overwrite'
335+
)
336+
# Set up the response to simulate a PreconditionFailed error
337+
self.http_response.status_code = 412
338+
self.parsed_responses = [
339+
{
340+
'Error': {
341+
'Code': 'PreconditionFailed',
342+
'Message': 'At least one of the pre-conditions you specified did not hold',
343+
'Condition': 'If-None-Match',
344+
}
345+
}
346+
]
347+
self.run_cmd(cmdline, expected_rc=0)
348+
# Verify PutObject was attempted with IfNoneMatch
349+
self.assertEqual(len(self.operations_called), 1)
350+
self.assertEqual(self.operations_called[0][0].name, 'PutObject')
351+
self.assertEqual(self.operations_called[0][1]['IfNoneMatch'], '*')
352+
# Verify source file was not deleted
353+
self.assertTrue(os.path.exists(full_path))
354+
355+
def test_mv_no_overwrite_flag_multipart_upload_when_object_not_exists_on_target(
356+
self,
357+
):
358+
# Create a large file that will trigger multipart upload
359+
full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024**2))
360+
cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
361+
# Set up responses for multipart upload
362+
self.parsed_responses = [
363+
{'UploadId': 'foo'}, # CreateMultipartUpload response
364+
{'ETag': '"foo-1"'}, # UploadPart response
365+
{'ETag': '"foo-2"'}, # UploadPart response
366+
{}, # CompleteMultipartUpload response
367+
]
368+
self.run_cmd(cmdline, expected_rc=0)
369+
# Verify all multipart operations were called
370+
self.assertEqual(len(self.operations_called), 4)
371+
self.assertEqual(
372+
self.operations_called[0][0].name, 'CreateMultipartUpload'
373+
)
374+
self.assertEqual(self.operations_called[1][0].name, 'UploadPart')
375+
self.assertEqual(self.operations_called[2][0].name, 'UploadPart')
376+
self.assertEqual(
377+
self.operations_called[3][0].name, 'CompleteMultipartUpload'
378+
)
379+
# Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
380+
self.assertEqual(self.operations_called[3][1]['IfNoneMatch'], '*')
381+
# Verify source file was deleted (successful move operation)
382+
self.assertFalse(os.path.exists(full_path))
383+
384+
def test_mv_no_overwrite_flag_multipart_upload_when_object_exists_on_target(
385+
self,
386+
):
387+
# Create a large file that will trigger multipart upload
388+
full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024**2))
389+
cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
390+
# Set up responses for multipart upload
391+
self.parsed_responses = [
392+
{'UploadId': 'foo'}, # CreateMultipartUpload response
393+
{'ETag': '"foo-1"'}, # UploadPart response
394+
{'ETag': '"foo-2"'}, # UploadPart response
395+
{
396+
'Error': {
397+
'Code': 'PreconditionFailed',
398+
'Message': 'At least one of the pre-conditions you specified did not hold',
399+
'Condition': 'If-None-Match',
400+
}
401+
}, # CompleteMultipartUpload response
402+
{}, # Abort Multipart
403+
]
404+
self.run_cmd(cmdline, expected_rc=0)
405+
# Set up the response to simulate a PreconditionFailed error
406+
self.http_response.status_code = 412
407+
# Verify all multipart operations were called
408+
self.assertEqual(len(self.operations_called), 5)
409+
self.assertEqual(
410+
self.operations_called[0][0].name, 'CreateMultipartUpload'
411+
)
412+
self.assertEqual(self.operations_called[1][0].name, 'UploadPart')
413+
self.assertEqual(self.operations_called[2][0].name, 'UploadPart')
414+
self.assertEqual(
415+
self.operations_called[3][0].name, 'CompleteMultipartUpload'
416+
)
417+
self.assertEqual(
418+
self.operations_called[4][0].name, 'AbortMultipartUpload'
419+
)
420+
# Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
421+
self.assertEqual(self.operations_called[3][1]['IfNoneMatch'], '*')
422+
# Verify source file was not deleted (failed move operation due to PreconditionFailed)
423+
self.assertTrue(os.path.exists(full_path))
424+
319425

320426
class TestMvWithCRTClient(BaseCRTTransferClientTest):
321427
def test_upload_move_using_crt_client(self):

‎tests/unit/customizations/s3/test_utils.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -859,6 +859,20 @@ def test_can_specify_amount_for_nonseekable_stream(self):
859859
self.assertEqual(nonseekable_fileobj.read(3), 'foo')
860860

861861

862+
class TestRequestParamsMapperNoOverwrite(unittest.TestCase):
863+
def test_map_put_object_params_with_no_overwrite(self):
864+
request_params = {}
865+
cli_params = {'no_overwrite': True}
866+
RequestParamsMapper.map_put_object_params(request_params, cli_params)
867+
self.assertEqual(request_params['IfNoneMatch'], '*')
868+
869+
def test_map_put_object_params_without_no_overwrite(self):
870+
request_params = {}
871+
cli_params = {}
872+
RequestParamsMapper.map_put_object_params(request_params, cli_params)
873+
self.assertNotIn('IfNoneMatch', request_params)
874+
875+
862876
class TestS3PathResolver:
863877
_BASE_ACCESSPOINT_ARN = (
864878
"s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint"

0 commit comments

Comments
 (0)