From 7f5a1de52dc54ac65bccaa4e3044632cab069406 Mon Sep 17 00:00:00 2001 From: Kevin DeJong Date: Sun, 1 Dec 2019 15:40:23 -0600 Subject: [PATCH] Enable integration testing as if cfnlint is a module --- test/integration/__init__.py | 28 ++++++++ test/integration/test_good_templates.py | 65 ++++++++++--------- test/integration/test_quickstart_templates.py | 8 ++- .../test_quickstart_templates_non_strict.py | 12 ++++ 4 files changed, 81 insertions(+), 32 deletions(-) diff --git a/test/integration/__init__.py b/test/integration/__init__.py index 3db7d5ba23..25906b1ef6 100644 --- a/test/integration/__init__.py +++ b/test/integration/__init__.py @@ -6,6 +6,7 @@ import json import subprocess from test.testlib.testcase import BaseTestCase +import cfnlint.core class BaseCliTestCase(BaseTestCase): @@ -61,3 +62,30 @@ def run_scenarios(self, extra_params=None): else: self.assertItemsEqual(expected_results, matches, 'Expected {} failures, got {} on {}'.format( len(expected_results), len(matches), filename)) + + def run_module_integration_scenarios(self, rules): + """ Test using cfnlint as a module integrated into another package""" + + cfnlint.core.configure_logging(None) + regions = ['us-east-1'] + for scenario in self.scenarios: + filename = scenario.get('filename') + results_filename = scenario.get('results_filename') + expected_results = scenario.get('results', []) + + if results_filename and not expected_results: + with open(results_filename) as json_data: + expected_results = json.load(json_data) + + template = cfnlint.decode.cfn_yaml.load(filename) + + matches = cfnlint.core.run_checks( + filename, + template, + rules, + regions + ) + + # Only check that the error count matches as the formats are different + self.assertEqual(len(expected_results), len(matches), 'Expected {} failures, got {} on {}'.format( + len(expected_results), len(matches), filename)) diff --git a/test/integration/test_good_templates.py b/test/integration/test_good_templates.py index fdf0da4624..876336aa5e 100644 --- a/test/integration/test_good_templates.py +++ b/test/integration/test_good_templates.py @@ -3,6 +3,7 @@ SPDX-License-Identifier: MIT-0 """ from test.integration import BaseCliTestCase +import cfnlint.core class TestQuickStartTemplates(BaseCliTestCase): @@ -10,23 +11,23 @@ class TestQuickStartTemplates(BaseCliTestCase): scenarios = [ { - "filename": 'test/fixtures/templates/good/generic.yaml', - "results": [], - "exit_code": 0, + 'filename': 'test/fixtures/templates/good/generic.yaml', + 'results': [], + 'exit_code': 0, }, { - "filename": 'test/fixtures/templates/good/minimal.yaml', - "results": [], - "exit_code": 0, + 'filename': 'test/fixtures/templates/good/minimal.yaml', + 'results': [], + 'exit_code': 0, }, { - "filename": 'test/fixtures/templates/good/transform.yaml', - "results": [], - "exit_code": 0, + 'filename': 'test/fixtures/templates/good/transform.yaml', + 'results': [], + 'exit_code': 0, }, { - "filename": 'test/fixtures/templates/bad/transform_serverless_template.yaml', - "results": [ + 'filename': 'test/fixtures/templates/bad/transform_serverless_template.yaml', + 'results': [ { 'Filename': 'test/fixtures/templates/bad/transform_serverless_template.yaml', 'Location': { @@ -94,31 +95,31 @@ class TestQuickStartTemplates(BaseCliTestCase): 'Message': "Error transforming template: Resource with id [myFunctionMyTimer] is invalid. Missing required property 'Schedule'." } ], - "exit_code": 2, + 'exit_code': 2, }, { - "filename": 'test/fixtures/templates/good/conditions.yaml', - "results": [], - "exit_code": 0, + 'filename': 'test/fixtures/templates/good/conditions.yaml', + 'results': [], + 'exit_code': 0, }, { 'filename': 'test/fixtures/templates/good/resources_codepipeline.yaml', - "results": [], - "exit_code": 0, + 'results': [], + 'exit_code': 0, }, { 'filename': 'test/fixtures/templates/good/transform_serverless_api.yaml', - "results": [], - "exit_code": 0, + 'results': [], + 'exit_code': 0, }, { 'filename': 'test/fixtures/templates/good/transform_serverless_function.yaml', - "results": [], - "exit_code": 0, + 'results': [], + 'exit_code': 0, }, { 'filename': 'test/fixtures/templates/good/transform_serverless_globals.yaml', - "results": [ + 'results': [ { 'Level': 'Error', 'Message': 'Deprecated runtime (nodejs6.10) specified. Updating disabled since 2019-08-12, please consider to update to nodejs10.x', @@ -136,25 +137,31 @@ class TestQuickStartTemplates(BaseCliTestCase): } } ], - "exit_code": 2, + 'exit_code': 2, }, { 'filename': 'test/fixtures/templates/good/transform/list_transform.yaml', - "results": [], - "exit_code": 0, + 'results': [], + 'exit_code': 0, }, { 'filename': 'test/fixtures/templates/good/transform/list_transform_many.yaml', - "results": [], - "exit_code": 0, + 'results': [], + 'exit_code': 0, }, { 'filename': 'test/fixtures/templates/good/transform/list_transform_not_sam.yaml', - "results": [], - "exit_code": 0, + 'results': [], + 'exit_code': 0, } ] def test_templates(self): """Test Successful JSON Parsing""" self.run_scenarios() + + def test_module_integration(self): + """ Test same templates using integration approach""" + rules = cfnlint.core.get_rules( + [], [], ['E', 'I', 'W'], {}, False) + self.run_module_integration_scenarios(rules) diff --git a/test/integration/test_quickstart_templates.py b/test/integration/test_quickstart_templates.py index c7824efc8a..77c3816efe 100644 --- a/test/integration/test_quickstart_templates.py +++ b/test/integration/test_quickstart_templates.py @@ -3,6 +3,7 @@ SPDX-License-Identifier: MIT-0 """ from test.integration import BaseCliTestCase +import cfnlint.core class TestQuickStartTemplates(BaseCliTestCase): @@ -72,6 +73,7 @@ class TestQuickStartTemplates(BaseCliTestCase): ] def test_templates(self): - """Test Successful JSON Parsing""" - self.maxDiff = None - self.run_scenarios(['--include-checks', 'I', '--include-expiremental']) + """ Test same templates using integration approach""" + rules = cfnlint.core.get_rules( + [], [], ['I', 'E', 'W'], {}, True) + self.run_module_integration_scenarios(rules) diff --git a/test/integration/test_quickstart_templates_non_strict.py b/test/integration/test_quickstart_templates_non_strict.py index e5fbb452c6..cb602997be 100644 --- a/test/integration/test_quickstart_templates_non_strict.py +++ b/test/integration/test_quickstart_templates_non_strict.py @@ -3,6 +3,7 @@ SPDX-License-Identifier: MIT-0 """ from test.integration import BaseCliTestCase +import cfnlint.core class TestQuickStartTemplates(BaseCliTestCase): @@ -39,3 +40,14 @@ def test_templates(self): '--include-expiremental', '--configure-rule', 'E3012:strict=false', ]) + + def test_module_integration(self): + """ Test same templates using integration approach""" + rules = cfnlint.core.get_rules( + [], [], ['I', 'E', 'W'], + { + 'E3012': { + 'strict': False, + } + }, True) + self.run_module_integration_scenarios(rules)