diff --git a/samcli/commands/validate/lib/sam_template_validator.py b/samcli/commands/validate/lib/sam_template_validator.py index ca27ac8c56..4c3421a981 100644 --- a/samcli/commands/validate/lib/sam_template_validator.py +++ b/samcli/commands/validate/lib/sam_template_validator.py @@ -3,6 +3,7 @@ """ import logging import functools +from pathlib import Path from samtranslator.public.exceptions import InvalidDocumentException from samtranslator.parser import parser @@ -10,8 +11,8 @@ from boto3.session import Session from samcli.lib.utils.packagetype import ZIP, IMAGE -from samcli.lib.utils.resources import AWS_SERVERLESS_FUNCTION -from samcli.yamlhelper import yaml_dump +from samcli.lib.utils.resources import AWS_SERVERLESS_FUNCTION, AWS_SERVERLESS_API, AWS_SERVERLESS_HTTPAPI +from samcli.yamlhelper import yaml_dump, parse_yaml_file from .exceptions import InvalidSamDocumentException LOG = logging.getLogger(__name__) @@ -65,6 +66,7 @@ def is_valid(self): self._replace_local_codeuri() self._replace_local_image() + self._replace_local_openapi() try: template = sam_translator.translate(sam_template=self.sam_template, parameter_values={}) @@ -139,6 +141,33 @@ def _replace_local_image(self): if "ImageUri" not in properties: properties["ImageUri"] = "111111111111.dkr.ecr.region.amazonaws.com/repository" + def _replace_local_openapi(self): + """ + Applies Transform Include of DefinitionBody. + Without this validation fails. + """ + # look for transform that includes a yaml definition like this: + # DefinitionBody: + # 'Fn::Transform': + # Name: AWS::Include + # Parameters: + # Location: openapi.yaml + resources = self.sam_template.get("Resources", {}) + for _, resource in resources.items(): + if not resource.get("Type", "") in [AWS_SERVERLESS_API, AWS_SERVERLESS_HTTPAPI]: + continue + + transform_dict = resource.get("Properties", {}).get("DefinitionBody", {}).get("Fn::Transform", {}) + if transform_dict.get("Name", "") != "AWS::Include": + continue + + location_prop = transform_dict.get("Parameters", {}).get("Location", "") + if not Path(location_prop).is_file(): + LOG.debug("Couldn't find file %s to import definition body", location_prop) + continue + + SamTemplateValidator._replace_with_file_contents(resource.get("Properties", {}), location_prop) + @staticmethod def is_s3_uri(uri): """ @@ -180,3 +209,11 @@ def _update_to_s3_uri(property_key, resource_property_dict, s3_uri_value="s3://b return resource_property_dict[property_key] = s3_uri_value + + @staticmethod + def _replace_with_file_contents(resource_property_dict, location_prop): + definition_body = parse_yaml_file(location_prop) + LOG.debug("Imported definition body from %s", location_prop) + + # replace the definition body from the file + resource_property_dict["DefinitionBody"] = definition_body diff --git a/tests/functional/commands/validate/lib/models/explicit_http_api_minimum_include.yaml b/tests/functional/commands/validate/lib/models/explicit_http_api_minimum_include.yaml new file mode 100644 index 0000000000..9ea5473e89 --- /dev/null +++ b/tests/functional/commands/validate/lib/models/explicit_http_api_minimum_include.yaml @@ -0,0 +1,18 @@ +Resources: + Api: + Type: AWS::Serverless::HttpApi + Properties: + CorsConfiguration: True + DefinitionBody: + 'Fn::Transform': + Name: AWS::Include + Parameters: + Location: tests/functional/commands/validate/lib/openapi/openapi.yaml + Function: + Type: AWS::Serverless::Function + Properties: + Handler: index.handler + Runtime: python3.7 + Events: + Api: + Type: HttpApi \ No newline at end of file diff --git a/tests/functional/commands/validate/lib/openapi/openapi.yaml b/tests/functional/commands/validate/lib/openapi/openapi.yaml new file mode 100644 index 0000000000..aea44b29ea --- /dev/null +++ b/tests/functional/commands/validate/lib/openapi/openapi.yaml @@ -0,0 +1,6 @@ +--- +openapi: 3.0.0 +paths: + /test: + get: + summary: Get test diff --git a/tests/integration/testdata/validate/openapi_json/openapi.json b/tests/integration/testdata/validate/openapi_json/openapi.json new file mode 100644 index 0000000000..0a67a8f6f5 --- /dev/null +++ b/tests/integration/testdata/validate/openapi_json/openapi.json @@ -0,0 +1,18 @@ +{ + "openapi": "3.0.0", + "info": { + "version": "1.0.0", + "title": "title" + }, + "paths": { + "/test": { + "get": { + "responses": { + "200": { + "description": "description" + } + } + } + } + } +} diff --git a/tests/integration/testdata/validate/openapi_json/template.json b/tests/integration/testdata/validate/openapi_json/template.json new file mode 100644 index 0000000000..251bb8911e --- /dev/null +++ b/tests/integration/testdata/validate/openapi_json/template.json @@ -0,0 +1,53 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessApi": { + "Type": "AWS::Serverless::Api", + "Properties": { + "StageName": "Prod", + "Auth": { + "DefaultAuthorizer": "Authorizer", + "Authorizers": { + "Authorizer": { + "AuthorizerPayloadFormatVersion": "2.0", + "FunctionArn": { + "Fn::ImportValue": "AuthorizerFunction" + } + } + } + }, + "DefinitionBody": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": {"Location": "./openapi.json"} + } + } + } + }, + "ServerlessHttpApi": { + "Type": "AWS::Serverless::HttpApi", + "Properties": { + "StageName": "Prod", + "Auth": { + "DefaultAuthorizer": "Authorizer", + "Authorizers": { + "Authorizer": { + "AuthorizerPayloadFormatVersion": "2.0", + "FunctionArn": { + "Fn::ImportValue": "AuthorizerFunction" + } + } + } + }, + "CorsConfiguration": true, + "DefinitionBody": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": {"Location": "./openapi.json"} + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/integration/testdata/validate/openapi_yaml/openapi.yaml b/tests/integration/testdata/validate/openapi_yaml/openapi.yaml new file mode 100644 index 0000000000..aea44b29ea --- /dev/null +++ b/tests/integration/testdata/validate/openapi_yaml/openapi.yaml @@ -0,0 +1,6 @@ +--- +openapi: 3.0.0 +paths: + /test: + get: + summary: Get test diff --git a/tests/integration/testdata/validate/openapi_yaml/template.yaml b/tests/integration/testdata/validate/openapi_yaml/template.yaml new file mode 100644 index 0000000000..f20740c4d9 --- /dev/null +++ b/tests/integration/testdata/validate/openapi_yaml/template.yaml @@ -0,0 +1,36 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: [AWS::Serverless-2016-10-31] +Resources: + ServerlessApi: + Type: AWS::Serverless::Api + Properties: + Auth: + DefaultAuthorizer: Authorizer + Authorizers: + Authorizer: + AuthorizerPayloadFormatVersion: '2.0' + FunctionArn: + Fn::ImportValue: AuthorizerFunction + DefinitionBody: + 'Fn::Transform': + Name: AWS::Include + Parameters: + Location: ./openapi.yaml + StageName: Prod + ServerlessHttpApi: + Type: AWS::Serverless::HttpApi + Properties: + Auth: + DefaultAuthorizer: Authorizer + Authorizers: + Authorizer: + AuthorizerPayloadFormatVersion: '2.0' + FunctionArn: + Fn::ImportValue: AuthorizerFunction + CorsConfiguration: True + DefinitionBody: + 'Fn::Transform': + Name: AWS::Include + Parameters: + Location: ./openapi.yaml + StageName: Prod diff --git a/tests/integration/validate/test_validate_command.py b/tests/integration/validate/test_validate_command.py index cba52002dd..f33b83be69 100644 --- a/tests/integration/validate/test_validate_command.py +++ b/tests/integration/validate/test_validate_command.py @@ -58,6 +58,8 @@ def command_list( [ ("default_yaml", TemplateFileTypes.YAML), # project with template.yaml ("default_json", TemplateFileTypes.JSON), # project with template.json + ("openapi_yaml", TemplateFileTypes.YAML), # project with template.yaml + ("openapi_json", TemplateFileTypes.JSON), # project with template.json ("multiple_files", TemplateFileTypes.YAML), # project with both template.yaml and template.json ( "with_build", diff --git a/tests/unit/commands/local/lib/swagger/test_parser.py b/tests/unit/commands/local/lib/swagger/test_parser.py index 7b9ea5758d..2d5a16017a 100644 --- a/tests/unit/commands/local/lib/swagger/test_parser.py +++ b/tests/unit/commands/local/lib/swagger/test_parser.py @@ -141,6 +141,39 @@ def test_payload_format_version(self): self.assertEqual(expected, result) + def test_payload_format_version_for_none(self): + function_name = "myfunction" + swagger = { + "paths": { + "/path1": {"get": {}}, + "/path2": {"get": {}}, + } + } + + parser = SwaggerParser(self.stack_path, swagger) + parser._get_integration_function_name = Mock() + parser._get_integration_function_name.return_value = function_name + + expected = [ + Route( + path="/path1", + methods=["get"], + function_name=function_name, + payload_format_version="None", + stack_path=self.stack_path, + ), + Route( + path="/path2", + methods=["get"], + function_name=function_name, + payload_format_version="None", + stack_path=self.stack_path, + ), + ] + result = parser.get_routes() + + self.assertEqual(expected, result) + @parameterized.expand( [ param("empty swagger", {}), diff --git a/tests/unit/commands/validate/lib/test_sam_template_validator.py b/tests/unit/commands/validate/lib/test_sam_template_validator.py index a269278b93..188689e61a 100644 --- a/tests/unit/commands/validate/lib/test_sam_template_validator.py +++ b/tests/unit/commands/validate/lib/test_sam_template_validator.py @@ -1,12 +1,20 @@ from unittest import TestCase from unittest.mock import Mock, patch -from samcli.lib.utils.packagetype import IMAGE +from samcli.lib.utils.packagetype import IMAGE, ZIP from samtranslator.public.exceptions import InvalidDocumentException from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.validate.lib.sam_template_validator import SamTemplateValidator +from samcli.yamlhelper import parse_yaml_file, yaml_parse +from pprint import pformat +from pathlib import Path + +import logging + +LOG = logging.getLogger(__name__) + class TestSamTemplateValidator(TestCase): @patch("samcli.commands.validate.lib.sam_template_validator.Session") @@ -120,6 +128,10 @@ def test_replace_local_codeuri(self): "Type": "AWS::Serverless::Api", "Properties": {"StageName": "Prod", "DefinitionUri": "./"}, }, + "ServerlessHttpApi": { + "Type": "AWS::Serverless::HttpApi", + "Properties": {"StageName": "Prod", "DefinitionUri": "./"}, + }, "ServerlessFunction": { "Type": "AWS::Serverless::Function", "Properties": {"Handler": "index.handler", "CodeUri": "./", "Runtime": "nodejs6.10", "Timeout": 60}, @@ -143,6 +155,9 @@ def test_replace_local_codeuri(self): self.assertEqual( template_resources.get("ServerlessApi").get("Properties").get("DefinitionUri"), "s3://bucket/value" ) + self.assertEqual( + template_resources.get("ServerlessHttpApi").get("Properties").get("DefinitionUri"), "s3://bucket/value" + ) self.assertEqual( template_resources.get("ServerlessFunction").get("Properties").get("CodeUri"), "s3://bucket/value" ) @@ -159,10 +174,15 @@ def test_replace_local_codeuri_when_no_codeuri_given(self): "Transform": "AWS::Serverless-2016-10-31", "Resources": { "ServerlessApi": {"Type": "AWS::Serverless::Api", "Properties": {"StageName": "Prod"}}, + "ServerlessHttpApi": {"Type": "AWS::Serverless::HttpApi", "Properties": {"StageName": "Prod"}}, "ServerlessFunction": { "Type": "AWS::Serverless::Function", "Properties": {"Handler": "index.handler", "Runtime": "nodejs6.10", "Timeout": 60}, }, + "ServerlessStateMachine": { + "Type": "AWS::Serverless::StateMachine", + "Properties": {"Role": "test-role-arn"}, + }, }, } @@ -173,9 +193,14 @@ def test_replace_local_codeuri_when_no_codeuri_given(self): validator._replace_local_codeuri() # check template - tempalte_resources = validator.sam_template.get("Resources") + template_resources = validator.sam_template.get("Resources") self.assertEqual( - tempalte_resources.get("ServerlessFunction").get("Properties").get("CodeUri"), "s3://bucket/value" + template_resources.get("ServerlessFunction").get("Properties").get("CodeUri"), "s3://bucket/value" + ) + self.assertEqual(template_resources.get("ServerlessApi").get("Properties").get("DefinitionUri", ""), "") + self.assertEqual(template_resources.get("ServerlessHttpApi").get("Properties").get("DefinitionUri", ""), "") + self.assertEqual( + template_resources.get("ServerlessStateMachine").get("Properties").get("DefinitionUri", ""), "" ) def test_dont_replace_local_codeuri_when_no_codeuri_given_packagetype_image(self): @@ -236,7 +261,10 @@ def test_dont_replace_codeuri_when_global_code_uri_given__both_packagetype(self) "Globals": { "Function": { "CodeUri": "s3://globalcodeuri", - } + }, + "Api": { + "Cors": "true", + }, }, "Resources": { "ServerlessApi": {"Type": "AWS::Serverless::Api", "Properties": {"StageName": "Prod"}}, @@ -268,6 +296,93 @@ def test_dont_replace_codeuri_when_global_code_uri_given__both_packagetype(self) template_resources.get("ServerlessFunctionZip").get("Properties").get("CodeUri"), "s3://bucket/value" ) + def test_replace_local_codeuri_in_global_section(self): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Globals": { + "Function": { + "CodeUri": "./my.zip", + }, + }, + "Resources": { + "ServerlessFunction": { + "Type": "AWS::Serverless::Function", + "Properties": {"PackageType": ZIP, "Timeout": 60}, + }, + }, + } + + managed_policy_mock = Mock() + + validator = SamTemplateValidator(template, managed_policy_mock) + + validator._replace_local_codeuri() + + # check template + global_resources = validator.sam_template.get("Resources") + self.assertEqual( + global_resources.get("ServerlessFunction").get("Properties").get("CodeUri"), "s3://bucket/value" + ) + + def test_replace_local_image_gets_replaced(self): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessApi": {"Type": "AWS::Serverless::Api", "Properties": {"StageName": "Prod"}}, + "ServerlessFunction": { + "Type": "AWS::Serverless::Function", + "Metadata": {"Dockerfile": "./Dockerfile"}, + "Properties": {"PackageType": IMAGE, "Timeout": 60}, + }, + }, + } + + managed_policy_mock = Mock() + + validator = SamTemplateValidator(template, managed_policy_mock) + + validator._replace_local_image() + + # check template + template_resources = validator.sam_template.get("Resources") + self.assertEqual( + template_resources.get("ServerlessFunction").get("Properties").get("ImageUri"), + "111111111111.dkr.ecr.region.amazonaws.com/repository", + ) + + def test_replace_local_image_doesnt_get_replaced_if_exists(self): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessApi": {"Type": "AWS::Serverless::Api", "Properties": {"StageName": "Prod"}}, + "ServerlessFunction": { + "Type": "AWS::Serverless::Function", + "Metadata": {"Dockerfile": "./Dockerfile"}, + "Properties": { + "PackageType": IMAGE, + "ImageUri": "222222222222.dkr.ecr.region.amazonaws.com/repository", + "Timeout": 60, + }, + }, + }, + } + + managed_policy_mock = Mock() + + validator = SamTemplateValidator(template, managed_policy_mock) + + validator._replace_local_image() + + # check template + template_resources = validator.sam_template.get("Resources") + self.assertEqual( + template_resources.get("ServerlessFunction").get("Properties").get("ImageUri"), + "222222222222.dkr.ecr.region.amazonaws.com/repository", + ) + def test_DefinitionUri_does_not_get_added_to_template_when_DefinitionBody_given(self): template = { "AWSTemplateFormatVersion": "2010-09-09", @@ -286,9 +401,9 @@ def test_DefinitionUri_does_not_get_added_to_template_when_DefinitionBody_given( validator._replace_local_codeuri() - tempalte_resources = validator.sam_template.get("Resources") - self.assertNotIn("DefinitionUri", tempalte_resources.get("ServerlessApi").get("Properties")) - self.assertIn("DefinitionBody", tempalte_resources.get("ServerlessApi").get("Properties")) + template_resources = validator.sam_template.get("Resources") + self.assertNotIn("DefinitionUri", template_resources.get("ServerlessApi").get("Properties")) + self.assertIn("DefinitionBody", template_resources.get("ServerlessApi").get("Properties")) def test_replace_local_codeuri_with_no_resources(self): @@ -306,3 +421,178 @@ def test_replace_local_codeuri_with_no_resources(self): # check template self.assertEqual(validator.sam_template.get("Resources"), {}) + + @patch("pathlib.Path.is_file") + @patch("samcli.commands.validate.lib.sam_template_validator.parse_yaml_file") + def test_DefinitionBody_gets_replaced_in_api(self, yaml_mock, path_mock): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessApi": { + "Type": "AWS::Serverless::Api", + "Properties": { + "StageName": "Prod", + "DefinitionBody": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": {"Location": "./tests/unit/commands/validate/lib/openapi/openapi.yaml"}, + } + }, + }, + }, + }, + } + openapi_yaml = """openapi: 3.0.0 +info: + version: "1.0.0" + title: title +paths: + '/test': + get: + responses: + 200: + description: description""" + + # mock file access + path_mock.return_value = True + yaml_mock.return_value = yaml_parse(openapi_yaml) + + managed_policy_mock = Mock() + + validator = SamTemplateValidator(template, managed_policy_mock) + + validator._replace_local_openapi() + + template_resources = validator.sam_template.get("Resources") + self.assertIn("DefinitionBody", template_resources.get("ServerlessApi").get("Properties")) + self.assertNotIn( + "Fn::Transform", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody") + ) + self.assertIn("openapi", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertIn("info", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertIn("paths", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + + @patch("pathlib.Path.is_file") + def test_DefinitionBody_not_replaced_if_file_not_found(self, path_mock): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessApi": { + "Type": "AWS::Serverless::HttpApi", + "Properties": { + "StageName": "Prod", + "DefinitionBody": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": {"Location": "./tests/unit/commands/validate/lib/openapi/notafile.yaml"}, + } + }, + }, + } + }, + } + + # mock file access + path_mock.return_value = False + + managed_policy_mock = Mock() + + validator = SamTemplateValidator(template, managed_policy_mock) + + validator._replace_local_openapi() + + template_resources = validator.sam_template.get("Resources") + self.assertIn("DefinitionBody", template_resources.get("ServerlessApi").get("Properties")) + self.assertIn("Fn::Transform", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertNotIn("openapi", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertNotIn("info", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertNotIn("paths", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + + @patch("pathlib.Path.is_file") + @patch("samcli.commands.validate.lib.sam_template_validator.parse_yaml_file") + def test_DefinitionBody_gets_replaced_if_json(self, yaml_mock, path_mock): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessApi": { + "Type": "AWS::Serverless::HttpApi", + "Properties": { + "StageName": "Prod", + "DefinitionBody": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": {"Location": "./tests/unit/commands/validate/lib/openapi/openapi.json"}, + } + }, + }, + } + }, + } + openapi_json = { + "openapi": "3.0.0", + "info": {"version": "1.0.0", "title": "title"}, + "paths": {"/test": {"get": {"responses": {"200": {"description": "description"}}}}}, + } + + # mock file access + path_mock.return_value = True + yaml_mock.return_value = openapi_json + + managed_policy_mock = Mock() + + validator = SamTemplateValidator(template, managed_policy_mock) + + validator._replace_local_openapi() + + template_resources = validator.sam_template.get("Resources") + self.assertIn("DefinitionBody", template_resources.get("ServerlessApi").get("Properties")) + self.assertNotIn( + "Fn::Transform", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody") + ) + self.assertIn("openapi", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertIn("info", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertIn("paths", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + + @patch("pathlib.Path.is_file") + @patch("samcli.commands.validate.lib.sam_template_validator.parse_yaml_file") + def test_DefinitionBody_not_replaced_if_not_include(self, yaml_mock, path_mock): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessApi": { + "Type": "AWS::Serverless::HttpApi", + "Properties": { + "StageName": "Prod", + "DefinitionBody": { + "Fn::Transform": { + "Name": "AWS::NotInclude", + } + }, + }, + } + }, + } + openapi_json = { + "openapi": "3.0.0", + "info": {"version": "1.0.0", "title": "title"}, + "paths": {"/test": {"get": {"responses": {"200": {"description": "description"}}}}}, + } + + # mock file access + path_mock.return_value = True + yaml_mock.side_effect = openapi_json + + managed_policy_mock = Mock() + + validator = SamTemplateValidator(template, managed_policy_mock) + + validator._replace_local_openapi() + + template_resources = validator.sam_template.get("Resources") + self.assertIn("DefinitionBody", template_resources.get("ServerlessApi").get("Properties")) + self.assertNotIn("openapi", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody")) + self.assertIn("Fn::Transform", template_resources.get("ServerlessApi").get("Properties").get("DefinitionBody"))