Add root_template_path to nested stack

samcli/commands/_utils/template.py

@@ -5,6 +5,7 @@
 import os
 import pathlib
 
+from typing import Optional
 import jmespath
 import yaml
 from botocore.utils import set_value_from_jmespath
@@ -29,20 +30,28 @@ class TemplateFailedParsingException(UserException):
     pass
 
 
-def get_template_data(template_file):
+def get_template_data(template_file, root_template_dir: Optional[str] = None):
     """
     Read the template file, parse it as JSON/YAML and return the template as a dictionary.
 
     Parameters
     ----------
     template_file : string
         Path to the template to read
+    root_template_dir: string
+        Optional directory of the root SAM Template
 
     Returns
     -------
     Template data as a dictionary
     """
 
+    # converting code path to absolute in normalize_resource_path will produce a bug
+    # it won't be resolved to correct value in resolve_code_path in codeuri.py later
+    # so we decided to convert it to absolute path here instead of in normalize_resource_path
+    if not os.path.isabs(template_file) and root_template_dir:
+        template_file = os.path.join(root_template_dir, template_file)
+
     if not pathlib.Path(template_file).exists():
         raise TemplateNotFoundException("Template file not found at {}".format(template_file))
 

samcli/commands/local/cli_common/invoke_context.py

@@ -404,6 +404,8 @@ def _get_stacks(self) -> List[Stack]:
                 self._template_file,
                 parameter_overrides=self._parameter_overrides,
                 global_parameter_overrides=self._global_parameter_overrides,
+                # root_template_dir will be used to convert relative path to absolute path
+                root_template_dir=os.path.dirname(os.path.abspath(self._template_file)),
             )
             return stacks
         except (TemplateNotFoundException, TemplateFailedParsingException) as ex:

samcli/lib/providers/sam_stack_provider.py

@@ -28,6 +28,7 @@ def __init__(
         template_dict: Dict,
         parameter_overrides: Optional[Dict] = None,
         global_parameter_overrides: Optional[Dict] = None,
+        root_template_dir: Optional[str] = None,
     ):
         """
         Initialize the class with SAM template data. The SAM template passed to this provider is assumed
@@ -43,6 +44,7 @@ def __init__(
             to get substituted within the template
         :param dict global_parameter_overrides: Optional dictionary of values for SAM template global parameters that
             might want to get substituted within the template and all its child templates
+        :param str root_template_dir: Optional directory of root SAM Template.
         """
 
         self._template_file = template_file
@@ -53,6 +55,7 @@ def __init__(
         )
         self._resources = self._template_dict.get("Resources", {})
         self._global_parameter_overrides = global_parameter_overrides
+        self._root_template_dir = root_template_dir
 
         LOG.debug("%d stacks found in the template", len(self._resources))
 
@@ -109,11 +112,19 @@ def _extract_stacks(self) -> None:
             try:
                 if resource_type == SamLocalStackProvider.SERVERLESS_APPLICATION:
                     stack = SamLocalStackProvider._convert_sam_application_resource(
-                        self._template_file, self._stack_path, name, resource_properties
+                        self._template_file,
+                        self._stack_path,
+                        name,
+                        resource_properties,
+                        root_template_dir=self._root_template_dir,
                     )
                 if resource_type == SamLocalStackProvider.CLOUDFORMATION_STACK:
                     stack = SamLocalStackProvider._convert_cfn_stack_resource(
-                        self._template_file, self._stack_path, name, resource_properties
+                        self._template_file,
+                        self._stack_path,
+                        name,
+                        resource_properties,
+                        root_template_dir=self._root_template_dir,
                     )
             except RemoteStackLocationNotSupported:
                 self.remote_stack_full_paths.append(get_full_path(self._stack_path, name))
@@ -129,6 +140,7 @@ def _convert_sam_application_resource(
         stack_path: str,
         name: str,
         resource_properties: Dict,
+        root_template_dir: Optional[str] = None,
         global_parameter_overrides: Optional[Dict] = None,
     ) -> Optional[Stack]:
         location = resource_properties.get("Location")
@@ -142,7 +154,7 @@ def _convert_sam_application_resource(
         if location.startswith("file://"):
             location = unquote(urlparse(location).path)
         else:
-            location = SamLocalStackProvider.normalize_resource_path(template_file, location)
+            location = SamLocalStackProvider.normalize_resource_path(template_file, location, root_template_dir)
 
         return Stack(
             parent_stack_path=stack_path,
@@ -151,7 +163,7 @@ def _convert_sam_application_resource(
             parameters=SamLocalStackProvider.merge_parameter_overrides(
                 resource_properties.get("Parameters", {}), global_parameter_overrides
             ),
-            template_dict=get_template_data(location),
+            template_dict=get_template_data(location, root_template_dir),
         )
 
     @staticmethod
@@ -160,6 +172,7 @@ def _convert_cfn_stack_resource(
         stack_path: str,
         name: str,
         resource_properties: Dict,
+        root_template_dir: Optional[str] = None,
         global_parameter_overrides: Optional[Dict] = None,
     ) -> Optional[Stack]:
         template_url = resource_properties.get("TemplateURL")
@@ -175,7 +188,7 @@ def _convert_cfn_stack_resource(
         if template_url.startswith("file://"):
             template_url = unquote(urlparse(template_url).path)
         else:
-            template_url = SamLocalStackProvider.normalize_resource_path(template_file, template_url)
+            template_url = SamLocalStackProvider.normalize_resource_path(template_file, template_url, root_template_dir)
 
         return Stack(
             parent_stack_path=stack_path,
@@ -184,7 +197,7 @@ def _convert_cfn_stack_resource(
             parameters=SamLocalStackProvider.merge_parameter_overrides(
                 resource_properties.get("Parameters", {}), global_parameter_overrides
             ),
-            template_dict=get_template_data(template_url),
+            template_dict=get_template_data(template_url, root_template_dir),
         )
 
     @staticmethod
@@ -194,6 +207,7 @@ def get_stacks(
         name: str = "",
         parameter_overrides: Optional[Dict] = None,
         global_parameter_overrides: Optional[Dict] = None,
+        root_template_dir: Optional[str] = None,
     ) -> Tuple[List[Stack], List[str]]:
         """
         Recursively extract stacks from a template file.
@@ -212,15 +226,16 @@ def get_stacks(
         global_parameter_overrides: Optional[Dict]
             Optional dictionary of values for SAM template global parameters
             that might want to get substituted within the template and its child templates
-
+        root_template_dir: str
+            Optional directory of root SAM Template. By default it is None unless we pass in a value.
         Returns
         -------
         stacks: List[Stack]
             The list of stacks extracted from template_file
         remote_stack_full_paths : List[str]
             The list of full paths of detected remote stacks
         """
-        template_dict = get_template_data(template_file)
+        template_dict = get_template_data(template_file, root_template_dir)
         stacks = [
             Stack(
                 stack_path,
@@ -233,7 +248,12 @@ def get_stacks(
         remote_stack_full_paths: List[str] = []
 
         current = SamLocalStackProvider(
-            template_file, stack_path, template_dict, parameter_overrides, global_parameter_overrides
+            template_file,
+            stack_path,
+            template_dict,
+            parameter_overrides,
+            global_parameter_overrides,
+            root_template_dir,
         )
         remote_stack_full_paths.extend(current.remote_stack_full_paths)
 
@@ -244,6 +264,7 @@ def get_stacks(
                 child_stack.name,
                 child_stack.parameters,
                 global_parameter_overrides,
+                root_template_dir=root_template_dir,
             )
             stacks.extend(stacks_in_child)
             remote_stack_full_paths.extend(remote_stack_full_paths_in_child)
@@ -290,7 +311,7 @@ def merge_parameter_overrides(
         return merged_parameter_overrides
 
     @staticmethod
-    def normalize_resource_path(stack_file_path: str, path: str) -> str:
+    def normalize_resource_path(stack_file_path: str, path: str, root_template_dir: Optional[str] = None) -> str:
         """
         Convert resource paths found in nested stack to ones resolvable from root stack.
         For example,
@@ -327,6 +348,8 @@ def normalize_resource_path(stack_file_path: str, path: str) -> str:
             The file path of the stack containing the resource
         path
             the raw path read from the template dict
+        root_template_dir
+            Optional directory of root SAM Template
 
         Returns
         -------
@@ -344,4 +367,10 @@ def normalize_resource_path(stack_file_path: str, path: str) -> str:
             # absolute paths are not robust as relative paths. So here prefer to use relative path.
             stack_file_path = os.path.relpath(os.path.realpath(stack_file_path))
 
+        # we would like to get relative code path if the input code path is relative (because of a bug)
+        # so it would get resolved to correct value using --docker-volume-basedir option later
+        # TODO: figure out is it possible to combine this with resolve_code_path in codeuri.py
+        if root_template_dir:
+            return os.path.relpath(os.path.join(os.path.dirname(stack_file_path), path), root_template_dir)
+
         return os.path.normpath(os.path.join(os.path.dirname(stack_file_path), path))

tests/integration/local/invoke/invoke_integ_base.py

@@ -21,6 +21,7 @@ def setUpClass(cls):
         cls.event_path = str(cls.test_data_path.joinpath("invoke", "event.json"))
         cls.event_utf8_path = str(cls.test_data_path.joinpath("invoke", "event_utf8.json"))
         cls.env_var_path = str(cls.test_data_path.joinpath("invoke", "vars.json"))
+        cls.docker_volume_basedir = str(cls.test_data_path.joinpath("invoke", "nested-templates"))
 
     @staticmethod
     def get_integ_dir():
@@ -46,6 +47,7 @@ def get_command_list(
         profile=None,
         layer_cache=None,
         docker_network=None,
+        docker_volume_basedir=None,
     ):
         command_list = [self.cmd, "local", "invoke", function_to_invoke]
 
@@ -79,6 +81,9 @@ def get_command_list(
         if region:
             command_list = command_list + ["--region", region]
 
+        if docker_volume_basedir:
+            command_list = command_list + ["--docker-volume-basedir", docker_volume_basedir]
+
         return command_list
 
     def get_build_command_list(

tests/integration/local/invoke/test_integrations_cli.py

@@ -458,6 +458,25 @@ def test_invoke_returns_expected_results_from_git_function_with_parameters(self)
         process_stdout = stdout.strip()
         self.assertEqual(process_stdout.decode("utf-8"), '"git init passed"')
 
+    @pytest.mark.flaky(reruns=3)
+    def test_invoke_with_docker_volume_basedir(self):
+        command_list = self.get_command_list(
+            "HelloWorldServerlessFunction",
+            template_path=self.template_path,
+            event_path=self.event_path,
+            docker_volume_basedir=self.docker_volume_basedir,
+        )
+
+        process = Popen(command_list, stdout=PIPE)
+        try:
+            stdout, _ = process.communicate(timeout=TIMEOUT)
+        except TimeoutExpired:
+            process.kill()
+            raise
+
+        process_stdout = stdout.strip()
+        self.assertEqual(process_stdout.decode("utf-8"), '"Hello world"')
+
 
 class TestSamInstrinsicsAndPlugins(InvokeIntegBase):
     template = Path("template-pseudo-params.yaml")

tests/unit/commands/local/cli_common/test_invoke_context.py

@@ -1082,5 +1082,8 @@ def test_must_pass_custom_region(self, get_stacks_mock):
         invoke_context = InvokeContext("template_file", aws_region="my-custom-region")
         invoke_context._get_stacks()
         get_stacks_mock.assert_called_with(
-            "template_file", parameter_overrides=None, global_parameter_overrides={"AWS::Region": "my-custom-region"}
+            "template_file",
+            parameter_overrides=None,
+            root_template_dir=os.getcwd(),  # using getcwd() because InvokeContext resolve relative path using cwd
+            global_parameter_overrides={"AWS::Region": "my-custom-region"},
         )

tests/unit/commands/local/lib/test_stack_provider.py

@@ -50,7 +50,7 @@ def test_sam_nested_stack_should_be_extracted(
                 }
             }
         }
-        self.get_template_data_mock.side_effect = lambda t: {
+        self.get_template_data_mock.side_effect = lambda t, m: {
             self.template_file: template,
             child_location_path: LEAF_TEMPLATE,
         }.get(t)
@@ -59,6 +59,7 @@ def test_sam_nested_stack_should_be_extracted(
             "",
             "",
             parameter_overrides=None,
+            root_template_dir=os.path.dirname(self.template_file),
         )
         self.assertListEqual(
             stacks,
@@ -88,16 +89,13 @@ def test_sam_deep_nested_stack(self):
                 }
             }
         }
-        self.get_template_data_mock.side_effect = lambda t: {
+        self.get_template_data_mock.side_effect = lambda t, m: {
             self.template_file: template,
             child_template_file: child_template,
             grand_child_template_file: LEAF_TEMPLATE,
         }.get(t)
         stacks, remote_stack_full_paths = SamLocalStackProvider.get_stacks(
-            self.template_file,
-            "",
-            "",
-            parameter_overrides=None,
+            self.template_file, "", "", parameter_overrides=None, root_template_dir=os.path.dirname(self.template_file)
         )
         self.assertListEqual(
             stacks,
@@ -119,7 +117,7 @@ def test_remote_stack_is_skipped(self, resource_type, location_property_name):
                 }
             }
         }
-        self.get_template_data_mock.side_effect = lambda t: {
+        self.get_template_data_mock.side_effect = lambda t, m: {
             self.template_file: template,
         }.get(t)
         stacks, remote_stack_full_paths = SamLocalStackProvider.get_stacks(
@@ -158,7 +156,7 @@ def test_sam_nested_stack_template_path_can_be_resolved_if_root_template_is_not_
                 }
             }
         }
-        self.get_template_data_mock.side_effect = lambda t: {
+        self.get_template_data_mock.side_effect = lambda t, m: {
             template_file: template,
             child_location_path: LEAF_TEMPLATE,
         }.get(t)
@@ -199,7 +197,7 @@ def test_global_parameter_overrides_can_be_passed_to_child_stacks(
                 }
             }
         }
-        self.get_template_data_mock.side_effect = lambda t: {
+        self.get_template_data_mock.side_effect = lambda t, m: {
             template_file: template,
             child_location_path: LEAF_TEMPLATE,
         }.get(t)