From 63f89c153573e28ad3b64a6dc1d01be9bfbfda91 Mon Sep 17 00:00:00 2001 From: JordonPhillips Date: Mon, 9 May 2016 15:39:43 -0700 Subject: [PATCH] Default ec2 page size to 1000 If this value isn't set, EC2 will not paginate at all. Instead, they will attempt to return all values to you at once. Naturally, this can cause significant delay if you have a ton of resources. --- .changes/next-release/bugfix-ec2.json | 5 ++ awscli/customizations/ec2/paginate.py | 27 ++++++++++ awscli/handlers.py | 2 + .../functional/ec2/test_describe_instances.py | 28 ++++++++-- tests/unit/customizations/ec2/__init__.py | 12 +++++ .../unit/customizations/ec2/test_paginate.py | 54 +++++++++++++++++++ 6 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 .changes/next-release/bugfix-ec2.json create mode 100644 awscli/customizations/ec2/paginate.py create mode 100644 tests/unit/customizations/ec2/__init__.py create mode 100644 tests/unit/customizations/ec2/test_paginate.py diff --git a/.changes/next-release/bugfix-ec2.json b/.changes/next-release/bugfix-ec2.json new file mode 100644 index 000000000000..2710df8cfd46 --- /dev/null +++ b/.changes/next-release/bugfix-ec2.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "ec2", + "description": "Sets MaxResults to default value of 1000." +} diff --git a/awscli/customizations/ec2/paginate.py b/awscli/customizations/ec2/paginate.py new file mode 100644 index 000000000000..5b23ad76f2da --- /dev/null +++ b/awscli/customizations/ec2/paginate.py @@ -0,0 +1,27 @@ +# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +DEFAULT_MAX_RESULTS = 1000 + + +def set_max_results_default(parsed_args, parsed_globals, **kwargs): + """ + In order to have EC2 return results you can paginate you need to inject + the `MaxItems` parameter. In the CLI we should be setting this by default + to avoid users having to wait exceedingly long times for the full results. + """ + + # The check for page size validates that the operation is a pagination + # operation. + if parsed_globals.paginate and hasattr(parsed_args, 'page_size') and \ + parsed_args.page_size is None and parsed_args.max_results is None: + parsed_args.page_size = DEFAULT_MAX_RESULTS diff --git a/awscli/handlers.py b/awscli/handlers.py index 3d3daed5cd96..075f7331ab6b 100644 --- a/awscli/handlers.py +++ b/awscli/handlers.py @@ -44,6 +44,7 @@ from awscli.customizations.ec2.protocolarg import register_protocol_args from awscli.customizations.ec2.runinstances import register_runinstances from awscli.customizations.ec2.secgroupsimplify import register_secgroup +from awscli.customizations.ec2.paginate import set_max_results_default from awscli.customizations.ecr import register_ecr_commands from awscli.customizations.emr.emr import emr_initialize from awscli.customizations.gamelift import register_gamelift_commands @@ -102,6 +103,7 @@ def awscli_initialize(event_handlers): ec2_add_priv_launch_key) register_parse_global_args(event_handlers) register_pagination(event_handlers) + event_handlers.register('operation-args-parsed.ec2.*', set_max_results_default) register_secgroup(event_handlers) register_bundleinstance(event_handlers) s3_plugin_initialize(event_handlers) diff --git a/tests/functional/ec2/test_describe_instances.py b/tests/functional/ec2/test_describe_instances.py index 38d7fba3b193..381831e3890b 100644 --- a/tests/functional/ec2/test_describe_instances.py +++ b/tests/functional/ec2/test_describe_instances.py @@ -11,6 +11,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from awscli.customizations.ec2.paginate import DEFAULT_MAX_RESULTS from awscli.testutils import BaseAWSCommandParamsTest @@ -20,26 +21,35 @@ class TestDescribeInstances(BaseAWSCommandParamsTest): def test_no_params(self): cmdline = self.prefix - result = {} + result = {'MaxResults': DEFAULT_MAX_RESULTS} self.assert_params_for_cmd(cmdline, result) def test_instance_id(self): args = ' --instance-ids i-12345678' cmdline = self.prefix + args - result = {'InstanceIds': ['i-12345678']} + result = { + 'InstanceIds': ['i-12345678'], + 'MaxResults': DEFAULT_MAX_RESULTS + } self.assert_params_for_cmd(cmdline, result) def test_instance_ids(self): args = ' --instance-ids i-12345678 i-87654321' cmdline = self.prefix + args - result = {'InstanceIds': ['i-12345678', 'i-87654321']} + result = { + 'InstanceIds': ['i-12345678', 'i-87654321'], + 'MaxResults': DEFAULT_MAX_RESULTS + } self.assert_params_for_cmd(cmdline, result) def test_instance_ids_alternate(self): # Not required, but will still work if you use JSON. args = ' --instance-ids ["i-12345678","i-87654321"]' cmdline = self.prefix + args - result = {'InstanceIds': ['i-12345678', 'i-87654321']} + result = { + 'InstanceIds': ['i-12345678', 'i-87654321'], + 'MaxResults': DEFAULT_MAX_RESULTS + } self.assert_params_for_cmd(cmdline, result) def test_filter_json(self): @@ -50,6 +60,7 @@ def test_filter_json(self): {'Name': 'group-name', 'Values': ['foobar']}, ], + 'MaxResults': DEFAULT_MAX_RESULTS } self.assert_params_for_cmd(cmdline, result) @@ -61,6 +72,7 @@ def test_filter_simple(self): {'Name': 'group-name', 'Values': ['foobar']}, ], + 'MaxResults': DEFAULT_MAX_RESULTS } self.assert_params_for_cmd(cmdline, result) @@ -72,6 +84,7 @@ def test_filter_values(self): {'Name': 'group-name', 'Values': ['foobar', 'fiebaz']}, ], + 'MaxResults': DEFAULT_MAX_RESULTS } self.assert_params_for_cmd(cmdline, result) @@ -86,6 +99,7 @@ def test_multiple_filters(self): {'Name': 'instance-id', 'Values': ['i-12345']}, ], + 'MaxResults': DEFAULT_MAX_RESULTS } self.assert_params_for_cmd(cmdline, result) @@ -101,6 +115,7 @@ def test_multiple_filters_alternate(self): {'Name': 'instance-id', 'Values': ['i-12345']}, ], + 'MaxResults': DEFAULT_MAX_RESULTS } self.assert_params_for_cmd(cmdlist, result) @@ -110,6 +125,11 @@ def test_page_size(self): result = {'MaxResults': 10} self.assert_params_for_cmd(cmdline, result) + def test_page_size_default(self): + cmdline = self.prefix + result = {'MaxResults': DEFAULT_MAX_RESULTS} + self.assert_params_for_cmd(cmdline, result) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/customizations/ec2/__init__.py b/tests/unit/customizations/ec2/__init__.py new file mode 100644 index 000000000000..cf69d677dd63 --- /dev/null +++ b/tests/unit/customizations/ec2/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/tests/unit/customizations/ec2/test_paginate.py b/tests/unit/customizations/ec2/test_paginate.py new file mode 100644 index 000000000000..6b6403b5717b --- /dev/null +++ b/tests/unit/customizations/ec2/test_paginate.py @@ -0,0 +1,54 @@ +# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import mock + +from awscli.customizations.ec2.paginate import set_max_results_default +from awscli.customizations.ec2.paginate import DEFAULT_MAX_RESULTS +from awscli.testutils import unittest + + +class TestSetMaxResult(unittest.TestCase): + def setUp(self): + self.parsed_args = mock.Mock() + self.parsed_globals = mock.Mock() + + self.parsed_args.max_results = None + self.parsed_args.page_size = None + self.parsed_globals.paginate = True + + def test_default_is_set(self): + set_max_results_default(self.parsed_args, self.parsed_globals) + self.assertEqual(self.parsed_args.page_size, DEFAULT_MAX_RESULTS) + + def test_page_size_isnt_overwritten(self): + page_size = DEFAULT_MAX_RESULTS - 10 + self.parsed_args.page_size = page_size + set_max_results_default(self.parsed_args, self.parsed_globals) + self.assertEqual(self.parsed_args.page_size, page_size) + + def test_max_results_isnt_overwritten(self): + max_results = DEFAULT_MAX_RESULTS - 10 + self.parsed_args.max_results = max_results + set_max_results_default(self.parsed_args, self.parsed_globals) + self.assertEqual(self.parsed_args.max_results, max_results) + self.assertEqual(self.parsed_args.page_size, None) + + def test_no_paginate_disables_default(self): + self.parsed_globals.paginate = False + set_max_results_default(self.parsed_args, self.parsed_globals) + self.assertEqual(self.parsed_args.page_size, None) + + def test_only_applies_if_page_size_is_present(self): + del self.parsed_args.page_size + set_max_results_default(self.parsed_args, self.parsed_globals) + self.assertFalse(hasattr(self.parsed_args, 'page_size'))