From 4d1e5ccb9d302a793b1a0f04858e4b8c45c59ff1 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Mon, 10 Apr 2023 18:48:19 +1000 Subject: [PATCH 01/33] Write rendered template to a temp file In the event that the reader class is unable to parse the rendered config, this adds logic to write the rendered config to a file to assist the caller in understand what went wrong. --- sceptre/config/reader.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sceptre/config/reader.py b/sceptre/config/reader.py index f618d87bd..2799543ac 100644 --- a/sceptre/config/reader.py +++ b/sceptre/config/reader.py @@ -14,6 +14,7 @@ import logging import sys import yaml +import tempfile from os import environ, path, walk from typing import Set, Tuple @@ -413,6 +414,16 @@ def _recursive_read( return config + def _write_debug_file(self, content): + with tempfile.NamedTemporaryFile( + mode="w", delete=False, prefix="rendered_" + ) as temp_file: + temp_file.write(content) + temp_file.flush() + + print(f"Debug file location: {temp_file.name}") + return temp_file.name + def _render(self, directory_path, basename, stack_group_config): """ Reads a configuration file, loads the config file as a template @@ -467,8 +478,11 @@ def _render(self, directory_path, basename, stack_group_config): try: config = yaml.safe_load(rendered_template) except Exception as err: + debug_file_path = self._write_debug_file(rendered_template) + raise ValueError( - "Error parsing {}:\n{}".format(abs_directory_path, err) + f"Error parsing {abs_directory_path}:\n{err}\n" + f"Rendered template saved to: {debug_file_path}" ) return config From c9ebb5ae5732b9346c86a78092ecdf536a852142 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 13:08:12 +1000 Subject: [PATCH 02/33] [Resolves #1318] Part 1 This adds a sceptre dump template command and deprecates sceptre generate. --- CONTRIBUTING.md | 7 +++---- sceptre/cli/dump.py | 42 +++++++++++++++++++++++++++++++++++++++++ sceptre/cli/template.py | 3 +++ venv.sh | 27 ++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) create mode 100755 venv.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87c0c8982..5561e5ab4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,11 +77,10 @@ $ git clone git@github.org:/sceptre.git 3. Install Sceptre for development (we recommend you use a [virtual environment](http://docs.python-guide.org/en/latest/dev/virtualenvs/)) +A convenient script is also provided: ```bash - $ cd sceptre/ - $ pip install -r requirements/prod.txt - $ pip install -r requirements/dev.txt - $ pip install -e . +$ cd sceptre +$ source venv.sh ``` 4. Create a branch for local development: diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index 2b2e885a2..8d993aa44 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -4,6 +4,8 @@ from sceptre.context import SceptreContext from sceptre.cli.helpers import catch_exceptions, write from sceptre.plan.plan import SceptrePlan +from sceptre.helpers import null_context +from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error logger = logging.getLogger(__name__) @@ -53,3 +55,43 @@ def dump_config(ctx, path): else: for config in output: write(config, output_format) + + +@dump_group.command(name="template") +@click.argument("path") +@click.option( + "-n", + "--no-placeholders", + is_flag=True, + help="If True, no placeholder values will be supplied for resolvers that cannot be resolved.", +) +@click.pass_context +@catch_exceptions +def dump_template(ctx, no_placeholders, path): + """ + Prints the template used for stack in PATH. + \f + + :param path: Path to execute the command on. + :type path: str + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + output_format=ctx.obj.get("output_format"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + ) + + plan = SceptrePlan(context) + + execution_context = ( + null_context() if no_placeholders else use_resolver_placeholders_on_error() + ) + with execution_context: + responses = plan.generate() + + output = [template for template in responses.values()] + write(output, context.output_format) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index 738900de8..05fc89709 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -2,7 +2,9 @@ import webbrowser import click +import deprecation +from sceptre import __version__ from sceptre.cli.helpers import catch_exceptions, write from sceptre.context import SceptreContext from sceptre.helpers import null_context @@ -55,6 +57,7 @@ def validate_command(ctx, no_placeholders, path): write(response, context.output_format) +@deprecation.deprecated("4.0.0", "5.0.0", __version__, "Use dump template instead.") @click.command(name="generate", short_help="Prints the template.") @click.option( "-n", diff --git a/venv.sh b/venv.sh new file mode 100755 index 000000000..6ccd7646c --- /dev/null +++ b/venv.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Also works with zsh. + +usage() { + echo "Usage: source $0" + echo "A script that sets up a Python Virtualenv" + exit 1 +} + +[ "$1" = "-h" ] && usage + +version="$(<.python-version)" +IFS="." read -r major minor _ <<< "$version" + +if ! python3 --version | grep -q "Python $major.$minor" ; then + echo "Please use pyenv and install Python $major.$minor" + return +fi + +virtualenv venv || \ + return + +. venv/bin/activate + +pip install -r requirements/prod.txt +pip install -r requirements/dev.txt +pip install -e . From 97d680f4e5ba2f4dc7affbdccbc13d485bb3142a Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 13:18:29 +1000 Subject: [PATCH 03/33] more --- sceptre/cli/dump.py | 18 +++++++++++++++--- sceptre/plan/actions.py | 7 +++++++ sceptre/plan/plan.py | 10 ++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index 8d993aa44..dee258ba5 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -91,7 +91,19 @@ def dump_template(ctx, no_placeholders, path): null_context() if no_placeholders else use_resolver_placeholders_on_error() ) with execution_context: - responses = plan.generate() + responses = plan.dump_template() - output = [template for template in responses.values()] - write(output, context.output_format) + output = [] + for stack, template in responses.items(): + if template is None: + logger.warning(f"{stack.external_name} does not exist") + else: + output.append({stack.external_name: template}) + + output_format = "json" if context.output_format == "json" else "yaml" + + if len(output) == 1: + write(output[0][stack.external_name], output_format) + else: + for template in output: + write(template, output_format) diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index bbcacdf53..d8bb02cee 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -1152,3 +1152,10 @@ def dump_config(self, config_reader: ConfigReader): """ stack_path = normalise_path(self.stack.name + ".yaml") return config_reader.read(stack_path) + + @add_stack_hooks + def dump_template(self): + """ + Returns the Template for the Stack + """ + return self.stack.template.body diff --git a/sceptre/plan/plan.py b/sceptre/plan/plan.py index 7b40dff1b..5da40c6f5 100644 --- a/sceptre/plan/plan.py +++ b/sceptre/plan/plan.py @@ -440,3 +440,13 @@ def dump_config(self, *args): """ self.resolve(command=self.dump_config.__name__) return self._execute(self.config_reader, *args) + + def dump_template(self, *args): + """ + Returns a generated Template for a given Stack + + :returns: A dictionary of Stacks and their template body. + :rtype: dict + """ + self.resolve(command=self.generate.__name__) + return self._execute(*args) From 3005b1424adaa424935d27544c5dab6bd9438ba0 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 14:08:47 +1000 Subject: [PATCH 04/33] version --- sceptre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/__init__.py b/sceptre/__init__.py index ba240b86e..2f84afcfa 100644 --- a/sceptre/__init__.py +++ b/sceptre/__init__.py @@ -7,7 +7,7 @@ __author__ = "Cloudreach" __email__ = "sceptre@cloudreach.com" -__version__ = "4.0.2" +__version__ = "4.1.0-alpha" # Set up logging to ``/dev/null`` like a library is supposed to. From acb399bb4c39a57cfad22d1f40359b472bc0096c Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 14:48:33 +1000 Subject: [PATCH 05/33] Revert "version" This reverts commit 3005b1424adaa424935d27544c5dab6bd9438ba0. --- sceptre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/__init__.py b/sceptre/__init__.py index 2f84afcfa..ba240b86e 100644 --- a/sceptre/__init__.py +++ b/sceptre/__init__.py @@ -7,7 +7,7 @@ __author__ = "Cloudreach" __email__ = "sceptre@cloudreach.com" -__version__ = "4.1.0-alpha" +__version__ = "4.0.2" # Set up logging to ``/dev/null`` like a library is supposed to. From a3bf4b7fb4aa055ee3583a10409fbd975f81bdfb Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 15:04:30 +1000 Subject: [PATCH 06/33] Revert deprecation --- sceptre/cli/template.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index 05fc89709..5608028aa 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -2,7 +2,6 @@ import webbrowser import click -import deprecation from sceptre import __version__ from sceptre.cli.helpers import catch_exceptions, write @@ -57,7 +56,6 @@ def validate_command(ctx, no_placeholders, path): write(response, context.output_format) -@deprecation.deprecated("4.0.0", "5.0.0", __version__, "Use dump template instead.") @click.command(name="generate", short_help="Prints the template.") @click.option( "-n", From c1ca8ed071eb0fb715354fed2ebeff4b9b1a6ef8 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 15:04:39 +1000 Subject: [PATCH 07/33] Revert "Revert "version"" This reverts commit acb399bb4c39a57cfad22d1f40359b472bc0096c. --- sceptre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/__init__.py b/sceptre/__init__.py index ba240b86e..2f84afcfa 100644 --- a/sceptre/__init__.py +++ b/sceptre/__init__.py @@ -7,7 +7,7 @@ __author__ = "Cloudreach" __email__ = "sceptre@cloudreach.com" -__version__ = "4.0.2" +__version__ = "4.1.0-alpha" # Set up logging to ``/dev/null`` like a library is supposed to. From bfa467513b69a42e1da46c09741c180a2b09a466 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 17:16:48 +1000 Subject: [PATCH 08/33] more --- sceptre/cli/dump.py | 11 +++++++++-- sceptre/cli/helpers.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index dee258ba5..6fd8e4336 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -7,6 +7,8 @@ from sceptre.helpers import null_context from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error +from pathlib import Path + logger = logging.getLogger(__name__) @@ -102,8 +104,13 @@ def dump_template(ctx, no_placeholders, path): output_format = "json" if context.output_format == "json" else "yaml" + def process_template(template): + stack_name = list(template.keys())[0] + file_path = Path(f".dump/template/{stack_name}/template.yaml") + write(template[stack_name], output_format, no_colour=True, file_path=file_path) + if len(output) == 1: - write(output[0][stack.external_name], output_format) + process_template(output[0]) else: for template in output: - write(template, output_format) + process_template(template) diff --git a/sceptre/cli/helpers.py b/sceptre/cli/helpers.py index 47efb87e2..a29d0a1a0 100644 --- a/sceptre/cli/helpers.py +++ b/sceptre/cli/helpers.py @@ -66,7 +66,7 @@ def confirmation(command, ignore, command_path, change_set=None): click.confirm(msg, abort=True) -def write(var, output_format="json", no_colour=True): +def write(var, output_format="json", no_colour=True, file_path=None): """ Writes ``var`` to stdout. If output_format is set to "json" or "yaml", write ``var`` as a JSON or YAML string. @@ -78,6 +78,8 @@ def write(var, output_format="json", no_colour=True): :type output_format: str :param no_colour: Whether to colour stack statuses :type no_colour: bool + :param file_path: Optional path to a file to save the output + :type file_path: str, optional """ output = var @@ -93,6 +95,13 @@ def write(var, output_format="json", no_colour=True): click.echo(output) + if file_path: + dir_path = file_path.parent + dir_path.mkdir(parents=True, exist_ok=True) + + with open(file_path, "w") as f: + f.write(output) + def _generate_json(stream): encoder = CustomJsonEncoder(indent=4) From fc2be81155aa9d9c07a90587b3b08bef8b94c95e Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 17:17:17 +1000 Subject: [PATCH 09/33] Revert "Write rendered template to a temp file" This reverts commit 4d1e5ccb9d302a793b1a0f04858e4b8c45c59ff1. --- sceptre/config/reader.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/sceptre/config/reader.py b/sceptre/config/reader.py index 2799543ac..f618d87bd 100644 --- a/sceptre/config/reader.py +++ b/sceptre/config/reader.py @@ -14,7 +14,6 @@ import logging import sys import yaml -import tempfile from os import environ, path, walk from typing import Set, Tuple @@ -414,16 +413,6 @@ def _recursive_read( return config - def _write_debug_file(self, content): - with tempfile.NamedTemporaryFile( - mode="w", delete=False, prefix="rendered_" - ) as temp_file: - temp_file.write(content) - temp_file.flush() - - print(f"Debug file location: {temp_file.name}") - return temp_file.name - def _render(self, directory_path, basename, stack_group_config): """ Reads a configuration file, loads the config file as a template @@ -478,11 +467,8 @@ def _render(self, directory_path, basename, stack_group_config): try: config = yaml.safe_load(rendered_template) except Exception as err: - debug_file_path = self._write_debug_file(rendered_template) - raise ValueError( - f"Error parsing {abs_directory_path}:\n{err}\n" - f"Rendered template saved to: {debug_file_path}" + "Error parsing {}:\n{}".format(abs_directory_path, err) ) return config From 8905933200a21cac0a8920be5fa4ceeb70d84ff8 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 17:18:49 +1000 Subject: [PATCH 10/33] Revert "Revert "Revert "version""" This reverts commit c1ca8ed071eb0fb715354fed2ebeff4b9b1a6ef8. --- sceptre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/__init__.py b/sceptre/__init__.py index 2f84afcfa..ba240b86e 100644 --- a/sceptre/__init__.py +++ b/sceptre/__init__.py @@ -7,7 +7,7 @@ __author__ = "Cloudreach" __email__ = "sceptre@cloudreach.com" -__version__ = "4.1.0-alpha" +__version__ = "4.0.2" # Set up logging to ``/dev/null`` like a library is supposed to. From 7c1d2a10e9c0d824db8ab6db91df3444a8b6aacf Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 18:00:27 +1000 Subject: [PATCH 11/33] more --- sceptre/cli/dump.py | 11 ++++++++--- sceptre/cli/template.py | 1 - 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index 6fd8e4336..220e90297 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -52,11 +52,16 @@ def dump_config(ctx, path): output_format = "json" if context.output_format == "json" else "yaml" + def process_config(config): + stack_name = list(config.keys())[0] + file_path = Path(f".dump/{stack_name}/config.yaml") + write(config[stack_name], output_format, no_colour=True, file_path=file_path) + if len(output) == 1: - write(output[0][stack.external_name], output_format) + process_config(output[0]) else: for config in output: - write(config, output_format) + process_config(config) @dump_group.command(name="template") @@ -106,7 +111,7 @@ def dump_template(ctx, no_placeholders, path): def process_template(template): stack_name = list(template.keys())[0] - file_path = Path(f".dump/template/{stack_name}/template.yaml") + file_path = Path(f".dump/{stack_name}/template.yaml") write(template[stack_name], output_format, no_colour=True, file_path=file_path) if len(output) == 1: diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index 5608028aa..738900de8 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -3,7 +3,6 @@ import click -from sceptre import __version__ from sceptre.cli.helpers import catch_exceptions, write from sceptre.context import SceptreContext from sceptre.helpers import null_context From 901c5cf84b9d2dcb04a222dbedce59496267a93b Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 18:21:10 +1000 Subject: [PATCH 12/33] more --- sceptre/cli/template.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index 05fc89709..fcd613307 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -2,9 +2,7 @@ import webbrowser import click -import deprecation -from sceptre import __version__ from sceptre.cli.helpers import catch_exceptions, write from sceptre.context import SceptreContext from sceptre.helpers import null_context From f9533bd99d494b005d5cf4cfdb04db6d681c9dee Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 15 Apr 2023 18:22:43 +1000 Subject: [PATCH 13/33] more --- sceptre/cli/template.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index fcd613307..738900de8 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -55,7 +55,6 @@ def validate_command(ctx, no_placeholders, path): write(response, context.output_format) -@deprecation.deprecated("4.0.0", "5.0.0", __version__, "Use dump template instead.") @click.command(name="generate", short_help="Prints the template.") @click.option( "-n", From efb20908f434a0e285a1fcbb5be92f2c39f9c406 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Tue, 18 Apr 2023 12:55:54 +1000 Subject: [PATCH 14/33] Restore CONTRIBUTING.md --- CONTRIBUTING.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5561e5ab4..87c0c8982 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,10 +77,11 @@ $ git clone git@github.org:/sceptre.git 3. Install Sceptre for development (we recommend you use a [virtual environment](http://docs.python-guide.org/en/latest/dev/virtualenvs/)) -A convenient script is also provided: ```bash -$ cd sceptre -$ source venv.sh + $ cd sceptre/ + $ pip install -r requirements/prod.txt + $ pip install -r requirements/dev.txt + $ pip install -e . ``` 4. Create a branch for local development: From 2ec723941d7c249e074f01ca855fe168ec84ff5f Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Tue, 18 Apr 2023 13:12:36 +1000 Subject: [PATCH 15/33] Remove venv.sh --- venv.sh | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100755 venv.sh diff --git a/venv.sh b/venv.sh deleted file mode 100755 index 6ccd7646c..000000000 --- a/venv.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# Also works with zsh. - -usage() { - echo "Usage: source $0" - echo "A script that sets up a Python Virtualenv" - exit 1 -} - -[ "$1" = "-h" ] && usage - -version="$(<.python-version)" -IFS="." read -r major minor _ <<< "$version" - -if ! python3 --version | grep -q "Python $major.$minor" ; then - echo "Please use pyenv and install Python $major.$minor" - return -fi - -virtualenv venv || \ - return - -. venv/bin/activate - -pip install -r requirements/prod.txt -pip install -r requirements/dev.txt -pip install -e . From c7cf881161532f7298edb8b8cd5f1195492420f1 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 21 Apr 2023 01:46:10 +1000 Subject: [PATCH 16/33] more --- sceptre/cli/helpers.py | 15 ++++++++++----- sceptre/plan/actions.py | 2 +- sceptre/plan/plan.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/sceptre/cli/helpers.py b/sceptre/cli/helpers.py index a29d0a1a0..74a0d6b4e 100644 --- a/sceptre/cli/helpers.py +++ b/sceptre/cli/helpers.py @@ -1,8 +1,12 @@ import logging import sys + from itertools import cycle from functools import partial, wraps +from typing import Any, Optional, Union +from pathlib import Path + import json import click import six @@ -66,20 +70,21 @@ def confirmation(command, ignore, command_path, change_set=None): click.confirm(msg, abort=True) -def write(var, output_format="json", no_colour=True, file_path=None): +def write( + var: Any, + output_format: str = "json", + no_colour: bool = True, + file_path: Optional[Path] = None, +) -> None: """ Writes ``var`` to stdout. If output_format is set to "json" or "yaml", write ``var`` as a JSON or YAML string. :param var: The object to print - :type var: object :param output_format: The format to print the output as. Allowed values: \ "text", "json", "yaml" - :type output_format: str :param no_colour: Whether to colour stack statuses - :type no_colour: bool :param file_path: Optional path to a file to save the output - :type file_path: str, optional """ output = var diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index d8bb02cee..6f469bbe7 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -629,7 +629,7 @@ def generate(self): """ Returns the Template for the Stack """ - return self.stack.template.body + return self.dump_template() @add_stack_hooks def validate(self): diff --git a/sceptre/plan/plan.py b/sceptre/plan/plan.py index 5da40c6f5..123b9f9c4 100644 --- a/sceptre/plan/plan.py +++ b/sceptre/plan/plan.py @@ -448,5 +448,5 @@ def dump_template(self, *args): :returns: A dictionary of Stacks and their template body. :rtype: dict """ - self.resolve(command=self.generate.__name__) + self.resolve(command=self.dump_template.__name__) return self._execute(*args) From b95a914523783a09e09c44b6685500d38b7d3820 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 21 Apr 2023 20:44:28 +1000 Subject: [PATCH 17/33] Deprecation --- sceptre/cli/template.py | 3 +++ sceptre/plan/actions.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index 738900de8..f71c7b53a 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -1,8 +1,10 @@ import logging import webbrowser +import deprecation import click +from sceptre import __version__ from sceptre.cli.helpers import catch_exceptions, write from sceptre.context import SceptreContext from sceptre.helpers import null_context @@ -65,6 +67,7 @@ def validate_command(ctx, no_placeholders, path): @click.argument("path") @click.pass_context @catch_exceptions +@deprecation.deprecated("4.0.0", "6.0.0", __version__, "Use dump template instead.") def generate_command(ctx, no_placeholders, path): """ Prints the template used for stack in PATH. diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 6f469bbe7..692416975 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -12,15 +12,17 @@ import time import typing import urllib -from datetime import datetime, timedelta -from os import path -from typing import Dict, Optional, Tuple, Union - import botocore +import deprecation + +from datetime import datetime, timedelta from dateutil.tz import tzutc +from os import path +from sceptre import __version__ from sceptre.config.reader import ConfigReader from sceptre.connection_manager import ConnectionManager + from sceptre.exceptions import ( CannotUpdateFailedStackError, ProtectedStackError, @@ -33,6 +35,8 @@ from sceptre.stack import Stack from sceptre.stack_status import StackChangeSetStatus, StackStatus +from typing import Dict, Optional, Tuple, Union + if typing.TYPE_CHECKING: from sceptre.diffing.stack_differ import StackDiff, StackDiffer @@ -624,6 +628,7 @@ def _convert_to_url(self, summaries): return new_summaries + @deprecation.deprecated("4.0.0", "6.0.0", __version__, "Use dump template instead.") @add_stack_hooks def generate(self): """ From 1b7a8d246b4e715379e508bfc0d3ae98995c7115 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 21 Apr 2023 21:23:55 +1000 Subject: [PATCH 18/33] docs --- docs/_source/docs/resolvers.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_source/docs/resolvers.rst b/docs/_source/docs/resolvers.rst index 97080123a..239e16eb5 100644 --- a/docs/_source/docs/resolvers.rst +++ b/docs/_source/docs/resolvers.rst @@ -476,7 +476,7 @@ have not yet been deployed. During normal deployment operations (using the ``lau ensure that order is followed, so everything works as expected. But there are other commands that will not actually deploy dependencies of a stack config before -operating on that Stack Config. These commands include ``generate``, ``validate``, and ``diff``. +operating on that Stack Config. These commands include ``dump template``, ``validate``, and ``diff``. If you have used resolvers to reverence other stacks, it is possible that a resolver might not be able to be resolved when performing that command's operations and will trigger an error. This is not likely to happen when you have only used resolvers in a stack's ``parameters``, but it is much more likely @@ -491,7 +491,7 @@ A few examples... and you run the ``diff`` command before other_stack.yaml has been deployed, the diff output will show the value of that parameter to be ``"{ !StackOutput(other_stack.yaml::OutputName) }"``. * If you have a ``sceptre_user_data`` value used in a Jinja template referencing - ``!stack_output other_stack.yaml::OutputName`` and you run the ``generate`` command, the generated + ``!stack_output other_stack.yaml::OutputName`` and you run the ``dump template`` command, the generated template will replace that value with ``"StackOutputotherstackyamlOutputName"``. This isn't as "pretty" as the sort of placeholder used for stack parameters, but the use of sceptre_user_data is broader, so it placeholder values can only be alphanumeric to reduce chances of it breaking the From a9a907656a1218f8d7c80542250980027da499d3 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 21 Apr 2023 21:44:14 +1000 Subject: [PATCH 19/33] Add dump all --- sceptre/cli/dump.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index 220e90297..63745eac3 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -54,7 +54,7 @@ def dump_config(ctx, path): def process_config(config): stack_name = list(config.keys())[0] - file_path = Path(f".dump/{stack_name}/config.yaml") + file_path = Path(".dump") / stack_name / f"config.{output_format}" write(config[stack_name], output_format, no_colour=True, file_path=file_path) if len(output) == 1: @@ -111,7 +111,7 @@ def dump_template(ctx, no_placeholders, path): def process_template(template): stack_name = list(template.keys())[0] - file_path = Path(f".dump/{stack_name}/template.yaml") + file_path = Path(".dump") / stack_name / f"template.{output_format}" write(template[stack_name], output_format, no_colour=True, file_path=file_path) if len(output) == 1: @@ -119,3 +119,25 @@ def process_template(template): else: for template in output: process_template(template) + + +@dump_group.command(name="all") +@click.argument("path") +@click.option( + "-n", + "--no-placeholders", + is_flag=True, + help="If True, no placeholder values will be supplied for resolvers that cannot be resolved.", +) +@click.pass_context +@catch_exceptions +def dump_all(ctx, no_placeholders, path): + """ + Dumps both the rendered (post-Jinja) Stack Configs and the template used for stack in PATH. + \f + + :param path: Path to execute the command on. + :type path: str + """ + ctx.invoke(dump_config, path=path) + ctx.invoke(dump_template, no_placeholders=no_placeholders, path=path) From d21f7dd6280e9e044170f598d09c5265e43aeb4f Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 21 Apr 2023 22:01:35 +1000 Subject: [PATCH 20/33] various fixes --- sceptre/cli/dump.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index 63745eac3..49697c73e 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -45,10 +45,7 @@ def dump_config(ctx, path): output = [] for stack, config in responses.items(): - if config is None: - logger.warning(f"{stack.external_name} does not exist") - else: - output.append({stack.external_name: config}) + output.append({stack.external_name: config}) output_format = "json" if context.output_format == "json" else "yaml" @@ -57,11 +54,8 @@ def process_config(config): file_path = Path(".dump") / stack_name / f"config.{output_format}" write(config[stack_name], output_format, no_colour=True, file_path=file_path) - if len(output) == 1: - process_config(output[0]) - else: - for config in output: - process_config(config) + for config in output: + process_config(config) @dump_group.command(name="template") @@ -102,10 +96,7 @@ def dump_template(ctx, no_placeholders, path): output = [] for stack, template in responses.items(): - if template is None: - logger.warning(f"{stack.external_name} does not exist") - else: - output.append({stack.external_name: template}) + output.append({stack.external_name: template}) output_format = "json" if context.output_format == "json" else "yaml" @@ -114,11 +105,8 @@ def process_template(template): file_path = Path(".dump") / stack_name / f"template.{output_format}" write(template[stack_name], output_format, no_colour=True, file_path=file_path) - if len(output) == 1: - process_template(output[0]) - else: - for template in output: - process_template(template) + for template in output: + process_template(template) @dump_group.command(name="all") From 7e8629b1bd66480ada0a85cb9c76628078d7686c Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 21 Apr 2023 22:30:12 +1000 Subject: [PATCH 21/33] Add info messages --- sceptre/cli/dump.py | 18 +++--------------- sceptre/cli/helpers.py | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index 49697c73e..ea9f5a965 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -2,13 +2,11 @@ import click from sceptre.context import SceptreContext -from sceptre.cli.helpers import catch_exceptions, write +from sceptre.cli.helpers import catch_exceptions, process_template from sceptre.plan.plan import SceptrePlan from sceptre.helpers import null_context from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error -from pathlib import Path - logger = logging.getLogger(__name__) @@ -49,13 +47,8 @@ def dump_config(ctx, path): output_format = "json" if context.output_format == "json" else "yaml" - def process_config(config): - stack_name = list(config.keys())[0] - file_path = Path(".dump") / stack_name / f"config.{output_format}" - write(config[stack_name], output_format, no_colour=True, file_path=file_path) - for config in output: - process_config(config) + process_template(config, output_format, type="config") @dump_group.command(name="template") @@ -100,13 +93,8 @@ def dump_template(ctx, no_placeholders, path): output_format = "json" if context.output_format == "json" else "yaml" - def process_template(template): - stack_name = list(template.keys())[0] - file_path = Path(".dump") / stack_name / f"template.{output_format}" - write(template[stack_name], output_format, no_colour=True, file_path=file_path) - for template in output: - process_template(template) + process_template(template, output_format) @dump_group.command(name="all") diff --git a/sceptre/cli/helpers.py b/sceptre/cli/helpers.py index 74a0d6b4e..95418a5b7 100644 --- a/sceptre/cli/helpers.py +++ b/sceptre/cli/helpers.py @@ -4,7 +4,7 @@ from itertools import cycle from functools import partial, wraps -from typing import Any, Optional, Union +from typing import Any, Optional from pathlib import Path import json @@ -20,6 +20,8 @@ from sceptre.stack_status import StackStatus from sceptre.stack_status_colourer import StackStatusColourer +logger = logging.getLogger(__name__) + def catch_exceptions(func): """ @@ -187,6 +189,23 @@ def _generate_text(stream): return stream +def process_template( + template: dict, output_format: str, type: str = "template" +) -> None: + """ + Helper to write templates and configs used in dump commands. + + :param template: the template. + :param output_format: The format to print the output as. Allowed values: \ + :param type: either template or config used only in the file name. + """ + stack_name = list(template.keys())[0] + file_path = Path(".dump") / stack_name / f"{type}.{output_format}" + logger.info(f"{stack_name} dumping {type} to {file_path}") + write(template[stack_name], output_format, no_colour=True, file_path=file_path) + logger.info(f"{stack_name} done.") + + def setup_vars(var_file, var, merge_vars, debug, no_colour): """ Handle --var-file and --var arguments before From 323e3e8e3c4134fa356a3f36942f2a438d319980 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Thu, 27 Apr 2023 19:05:08 +1000 Subject: [PATCH 22/33] Update sceptre/plan/actions.py Co-authored-by: Jon Falkenstein <77296393+jfalkenstein@users.noreply.github.com> --- sceptre/plan/actions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 692416975..6d74a144b 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -628,8 +628,7 @@ def _convert_to_url(self, summaries): return new_summaries - @deprecation.deprecated("4.0.0", "6.0.0", __version__, "Use dump template instead.") - @add_stack_hooks + @deprecation.deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") def generate(self): """ Returns the Template for the Stack From e755dbf0f8f965144dd7b0d5dbb07dab3ef261d3 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Thu, 27 Apr 2023 21:30:22 +1000 Subject: [PATCH 23/33] Fix names --- sceptre/cli/dump.py | 6 +++--- sceptre/cli/helpers.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index ea9f5a965..87bdec1c8 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -2,7 +2,7 @@ import click from sceptre.context import SceptreContext -from sceptre.cli.helpers import catch_exceptions, process_template +from sceptre.cli.helpers import catch_exceptions, dump_to_file from sceptre.plan.plan import SceptrePlan from sceptre.helpers import null_context from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error @@ -48,7 +48,7 @@ def dump_config(ctx, path): output_format = "json" if context.output_format == "json" else "yaml" for config in output: - process_template(config, output_format, type="config") + dump_to_file(config, output_format, type="config") @dump_group.command(name="template") @@ -94,7 +94,7 @@ def dump_template(ctx, no_placeholders, path): output_format = "json" if context.output_format == "json" else "yaml" for template in output: - process_template(template, output_format) + dump_to_file(template, output_format) @dump_group.command(name="all") diff --git a/sceptre/cli/helpers.py b/sceptre/cli/helpers.py index 95418a5b7..413dd8085 100644 --- a/sceptre/cli/helpers.py +++ b/sceptre/cli/helpers.py @@ -189,21 +189,21 @@ def _generate_text(stream): return stream -def process_template( - template: dict, output_format: str, type: str = "template" +def dump_to_file( + template: dict, output_format: str, template_type: str = "template" ) -> None: """ Helper to write templates and configs used in dump commands. :param template: the template. :param output_format: The format to print the output as. Allowed values: \ - :param type: either template or config used only in the file name. + :param template_type: either template or config used only in the file name. """ stack_name = list(template.keys())[0] - file_path = Path(".dump") / stack_name / f"{type}.{output_format}" - logger.info(f"{stack_name} dumping {type} to {file_path}") + file_path = Path(".dump") / stack_name / f"{template_type}.{output_format}" + logger.info(f"{stack_name} dumping {template_type} to {file_path}") write(template[stack_name], output_format, no_colour=True, file_path=file_path) - logger.info(f"{stack_name} done.") + logger.info(f"{stack_name} dump to {file_path} complete.") def setup_vars(var_file, var, merge_vars, debug, no_colour): From 22a0516f66182f96961c1dee17e52553a33f52d0 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Thu, 27 Apr 2023 21:34:32 +1000 Subject: [PATCH 24/33] Update sceptre/cli/template.py Co-authored-by: Jon Falkenstein <77296393+jfalkenstein@users.noreply.github.com> --- sceptre/cli/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index f71c7b53a..4b22c9319 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -67,7 +67,7 @@ def validate_command(ctx, no_placeholders, path): @click.argument("path") @click.pass_context @catch_exceptions -@deprecation.deprecated("4.0.0", "6.0.0", __version__, "Use dump template instead.") +@deprecation.deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") def generate_command(ctx, no_placeholders, path): """ Prints the template used for stack in PATH. From 5bbfb6fcf31c5ec5ae496a2c020cbf1cafbfca5f Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Thu, 27 Apr 2023 22:49:50 +1000 Subject: [PATCH 25/33] fail_if_not_removed --- sceptre/cli/template.py | 6 +++--- sceptre/plan/actions.py | 4 ++-- sceptre/plan/plan.py | 4 ++++ sceptre/stack.py | 8 ++++---- tests/test_connection_manager.py | 9 +++++---- tests/test_plan.py | 8 ++++++++ tests/test_stack.py | 8 ++++---- 7 files changed, 30 insertions(+), 17 deletions(-) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index 4b22c9319..cf81c1b2d 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -1,9 +1,9 @@ import logging import webbrowser -import deprecation - import click +from deprecation import deprecated + from sceptre import __version__ from sceptre.cli.helpers import catch_exceptions, write from sceptre.context import SceptreContext @@ -67,7 +67,7 @@ def validate_command(ctx, no_placeholders, path): @click.argument("path") @click.pass_context @catch_exceptions -@deprecation.deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") +@deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") def generate_command(ctx, no_placeholders, path): """ Prints the template used for stack in PATH. diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 6d74a144b..5d61c958a 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -13,11 +13,11 @@ import typing import urllib import botocore -import deprecation from datetime import datetime, timedelta from dateutil.tz import tzutc from os import path +from deprecation import deprecated from sceptre import __version__ from sceptre.config.reader import ConfigReader @@ -628,7 +628,7 @@ def _convert_to_url(self, summaries): return new_summaries - @deprecation.deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") + @deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") def generate(self): """ Returns the Template for the Stack diff --git a/sceptre/plan/plan.py b/sceptre/plan/plan.py index 123b9f9c4..a82a37555 100644 --- a/sceptre/plan/plan.py +++ b/sceptre/plan/plan.py @@ -8,8 +8,10 @@ """ import functools import itertools + from os import path, walk from typing import Dict, List, Set, Callable, Iterable, Optional +from deprecation import deprecated from sceptre.config.graph import StackGraph from sceptre.config.reader import ConfigReader @@ -19,6 +21,7 @@ from sceptre.helpers import sceptreise_path from sceptre.plan.executor import SceptrePlanExecutor from sceptre.stack import Stack +from sceptre import __version__ def require_resolved(func) -> Callable: @@ -441,6 +444,7 @@ def dump_config(self, *args): self.resolve(command=self.dump_config.__name__) return self._execute(self.config_reader, *args) + @deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") def dump_template(self, *args): """ Returns a generated Template for a given Stack diff --git a/sceptre/stack.py b/sceptre/stack.py index b953dcb94..251c46a24 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -8,9 +8,9 @@ """ import logging -from typing import List, Any, Optional -import deprecation +from typing import List, Any, Optional +from deprecation import deprecated from sceptre import __version__ from sceptre.connection_manager import ConnectionManager @@ -383,7 +383,7 @@ def template(self): return self._template @property - @deprecation.deprecated( + @deprecated( "4.0.0", "5.0.0", __version__, "Use the template Stack Config key instead." ) def template_path(self) -> str: @@ -393,7 +393,7 @@ def template_path(self) -> str: return self.template_handler_config["path"] @template_path.setter - @deprecation.deprecated( + @deprecated( "4.0.0", "5.0.0", __version__, "Use the template Stack Config key instead." ) def template_path(self, value: str): diff --git a/tests/test_connection_manager.py b/tests/test_connection_manager.py index 71bc21621..e2a4aa9e4 100644 --- a/tests/test_connection_manager.py +++ b/tests/test_connection_manager.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- import warnings +import pytest + from collections import defaultdict from typing import Union from unittest.mock import Mock, patch, sentinel, create_autospec +from deprecation import fail_if_not_removed -import deprecation -import pytest from boto3.session import Session from botocore.exceptions import ClientError from moto import mock_s3 @@ -805,11 +806,11 @@ def test_create_session_environment_variables__include_system_envs_false__does_n } assert expected == result - @deprecation.fail_if_not_removed + @fail_if_not_removed def test_iam_role__is_removed_on_removal_version(self): self.connection_manager.iam_role - @deprecation.fail_if_not_removed + @fail_if_not_removed def test_iam_role_session_duration__is_removed_on_removal_version(self): self.connection_manager.iam_role_session_duration diff --git a/tests/test_plan.py b/tests/test_plan.py index 9e92ba9ab..11f0a4eb0 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -1,6 +1,8 @@ import pytest from unittest.mock import MagicMock, patch, sentinel +from deprecation import fail_if_not_removed + from sceptre.context import SceptreContext from sceptre.stack import Stack from sceptre.config.reader import ConfigReader @@ -61,3 +63,9 @@ def test_command_not_found_error_raised(self): plan = MagicMock(spec=SceptrePlan) plan.context = self.mock_context plan.invalid_command() + + @fail_if_not_removed + def test_generate_removed(self): + plan = MagicMock(spec=SceptrePlan) + plan.context = self.mock_context + plan.generate("test-attribute") diff --git a/tests/test_stack.py b/tests/test_stack.py index 4683d0073..7e843e47f 100644 --- a/tests/test_stack.py +++ b/tests/test_stack.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from unittest.mock import MagicMock, sentinel +from deprecation import fail_if_not_removed -import deprecation import pytest from sceptre.exceptions import InvalidConfigFileError @@ -257,15 +257,15 @@ def resolve(self): connection_manager = self.stack.connection_manager assert connection_manager.sceptre_role == "role" - @deprecation.fail_if_not_removed + @fail_if_not_removed def test_iam_role__is_removed_on_removal_version(self): self.stack.iam_role - @deprecation.fail_if_not_removed + @fail_if_not_removed def test_role_arn__is_removed_on_removal_version(self): self.stack.role_arn - @deprecation.fail_if_not_removed + @fail_if_not_removed def test_iam_role_session_duration__is_removed_on_removal_version(self): self.stack.iam_role_session_duration From 119162de4bdf36d164bf6822ca96076e193b5f80 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 28 Apr 2023 13:56:10 +1000 Subject: [PATCH 26/33] more --- integration-tests/steps/templates.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration-tests/steps/templates.py b/integration-tests/steps/templates.py index 98ef3473d..cb62480cf 100644 --- a/integration-tests/steps/templates.py +++ b/integration-tests/steps/templates.py @@ -77,7 +77,7 @@ def step_impl(context, stack_name): context.error = e -@when('the user generates the template for stack "{stack_name}"') +@when('the user dumps the template for stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( command_path=stack_name + ".yaml", project_path=context.sceptre_dir @@ -85,6 +85,7 @@ def step_impl(context, stack_name): config_path = sceptre_context.full_config_path() template_path = sceptre_context.full_templates_path() + with open(os.path.join(config_path, stack_name + ".yaml")) as config_file: stack_config = yaml.safe_load(config_file) @@ -100,8 +101,9 @@ def step_impl(context, stack_name): stack_config = yaml.safe_load(config_file) sceptre_plan = SceptrePlan(sceptre_context) + try: - context.output = sceptre_plan.generate() + context.output = sceptre_plan.dump_template() except Exception as e: context.error = e From 94dcaf1a4a9f9353c11e15f50dd2218ee213996d Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Tue, 2 May 2023 17:52:30 +1000 Subject: [PATCH 27/33] Correction to deprecation --- sceptre/plan/actions.py | 2 -- sceptre/plan/plan.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index ba173e1a4..ac9c60e3c 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -20,8 +20,6 @@ from deprecation import deprecated from sceptre import __version__ -from sceptre.config.reader import ConfigReader - from sceptre.connection_manager import ConnectionManager from sceptre.exceptions import ( diff --git a/sceptre/plan/plan.py b/sceptre/plan/plan.py index e5d38ce40..4644a2988 100644 --- a/sceptre/plan/plan.py +++ b/sceptre/plan/plan.py @@ -380,6 +380,7 @@ def estimate_cost(self, *args): self.resolve(command=self.estimate_cost.__name__) return self._execute(*args) + @deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") def generate(self, *args): """ Returns a generated Template for a given Stack @@ -444,7 +445,6 @@ def dump_config(self, *args): self.resolve(command=self.dump_config.__name__) return self._execute(*args) - @deprecated("4.2.0", "5.0.0", __version__, "Use dump template instead.") def dump_template(self, *args): """ Returns a generated Template for a given Stack From 3e9682ebb82c2944bc8f18aaa18a4d2d980686ab Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Tue, 2 May 2023 18:39:58 +1000 Subject: [PATCH 28/33] Add to-file --- sceptre/cli/dump.py | 45 +++++++++++++++++++++++++++++------------- sceptre/cli/helpers.py | 17 ---------------- sceptre/plan/plan.py | 4 +--- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index 87bdec1c8..b8f58b622 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -1,8 +1,10 @@ import logging import click +from pathlib import Path + from sceptre.context import SceptreContext -from sceptre.cli.helpers import catch_exceptions, dump_to_file +from sceptre.cli.helpers import catch_exceptions, write from sceptre.plan.plan import SceptrePlan from sceptre.helpers import null_context from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error @@ -20,9 +22,12 @@ def dump_group(): @dump_group.command(name="config") @click.argument("path") +@click.option( + "--to-file", is_flag=True, help="If True, also dump the template to a local file." +) @click.pass_context @catch_exceptions -def dump_config(ctx, path): +def dump_config(ctx, to_file, path): """ Dump the rendered (post-Jinja) Stack Configs. \f @@ -41,14 +46,19 @@ def dump_config(ctx, path): plan = SceptrePlan(context) responses = plan.dump_config() - output = [] + output_format = "json" if context.output_format == "json" else "yaml" + for stack, config in responses.items(): - output.append({stack.external_name: config}) + stack_name = stack.external_name - output_format = "json" if context.output_format == "json" else "yaml" + if to_file: + file_path = Path(".dump") / stack_name / f"config.{output_format}" + logger.info(f"{stack_name} dumping to {file_path}") + write(config, output_format, no_colour=True, file_path=file_path) + logger.info(f"{stack_name} dump to {file_path} complete.") - for config in output: - dump_to_file(config, output_format, type="config") + else: + write(config, output_format, no_colour=True) @dump_group.command(name="template") @@ -59,9 +69,12 @@ def dump_config(ctx, path): is_flag=True, help="If True, no placeholder values will be supplied for resolvers that cannot be resolved.", ) +@click.option( + "--to-file", is_flag=True, help="If True, also dump the template to a local file." +) @click.pass_context @catch_exceptions -def dump_template(ctx, no_placeholders, path): +def dump_template(ctx, to_file, no_placeholders, path): """ Prints the template used for stack in PATH. \f @@ -78,7 +91,6 @@ def dump_template(ctx, no_placeholders, path): output_format=ctx.obj.get("output_format"), ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) - plan = SceptrePlan(context) execution_context = ( @@ -87,14 +99,19 @@ def dump_template(ctx, no_placeholders, path): with execution_context: responses = plan.dump_template() - output = [] + output_format = "json" if context.output_format == "json" else "yaml" + for stack, template in responses.items(): - output.append({stack.external_name: template}) + stack_name = stack.external_name - output_format = "json" if context.output_format == "json" else "yaml" + if to_file: + file_path = Path(".dump") / stack_name / f"template.{output_format}" + logger.info(f"{stack_name} dumping template to {file_path}") + write(template, output_format, no_colour=True, file_path=file_path) + logger.info(f"{stack_name} dump to {file_path} complete.") - for template in output: - dump_to_file(template, output_format) + else: + write(template, output_format, no_colour=True) @dump_group.command(name="all") diff --git a/sceptre/cli/helpers.py b/sceptre/cli/helpers.py index 413dd8085..1fa7b7ec8 100644 --- a/sceptre/cli/helpers.py +++ b/sceptre/cli/helpers.py @@ -189,23 +189,6 @@ def _generate_text(stream): return stream -def dump_to_file( - template: dict, output_format: str, template_type: str = "template" -) -> None: - """ - Helper to write templates and configs used in dump commands. - - :param template: the template. - :param output_format: The format to print the output as. Allowed values: \ - :param template_type: either template or config used only in the file name. - """ - stack_name = list(template.keys())[0] - file_path = Path(".dump") / stack_name / f"{template_type}.{output_format}" - logger.info(f"{stack_name} dumping {template_type} to {file_path}") - write(template[stack_name], output_format, no_colour=True, file_path=file_path) - logger.info(f"{stack_name} dump to {file_path} complete.") - - def setup_vars(var_file, var, merge_vars, debug, no_colour): """ Handle --var-file and --var arguments before diff --git a/sceptre/plan/plan.py b/sceptre/plan/plan.py index 4644a2988..46008afab 100644 --- a/sceptre/plan/plan.py +++ b/sceptre/plan/plan.py @@ -448,8 +448,6 @@ def dump_config(self, *args): def dump_template(self, *args): """ Returns a generated Template for a given Stack - - :returns: A dictionary of Stacks and their template body. - :rtype: dict """ self.resolve(command=self.dump_template.__name__) + return self._execute(*args) From e5321c089a9321afb2eaad121b1c1460c6c5aef1 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Thu, 4 May 2023 08:58:33 +1000 Subject: [PATCH 29/33] Correction in steps --- integration-tests/steps/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/steps/templates.py b/integration-tests/steps/templates.py index cb62480cf..dfa49c9de 100644 --- a/integration-tests/steps/templates.py +++ b/integration-tests/steps/templates.py @@ -119,7 +119,7 @@ def step_impl(context, stack_name): ) sceptre_plan = SceptrePlan(sceptre_context) try: - context.output = sceptre_plan.generate() + context.output = sceptre_plan.dump_template() except Exception as e: context.error = e From d08f091dcaa04b319b8d393b2910c228bbeedbe5 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sun, 7 May 2023 22:30:19 +1000 Subject: [PATCH 30/33] Respone to reviews --- sceptre/cli/dump.py | 11 ++++++++--- sceptre/cli/helpers.py | 13 ++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py index b8f58b622..e26709445 100644 --- a/sceptre/cli/dump.py +++ b/sceptre/cli/dump.py @@ -122,9 +122,12 @@ def dump_template(ctx, to_file, no_placeholders, path): is_flag=True, help="If True, no placeholder values will be supplied for resolvers that cannot be resolved.", ) +@click.option( + "--to-file", is_flag=True, help="If True, also dump the template to a local file." +) @click.pass_context @catch_exceptions -def dump_all(ctx, no_placeholders, path): +def dump_all(ctx, to_file, no_placeholders, path): """ Dumps both the rendered (post-Jinja) Stack Configs and the template used for stack in PATH. \f @@ -132,5 +135,7 @@ def dump_all(ctx, no_placeholders, path): :param path: Path to execute the command on. :type path: str """ - ctx.invoke(dump_config, path=path) - ctx.invoke(dump_template, no_placeholders=no_placeholders, path=path) + ctx.invoke(dump_config, to_file=to_file, path=path) + ctx.invoke( + dump_template, to_file=to_file, no_placeholders=no_placeholders, path=path + ) diff --git a/sceptre/cli/helpers.py b/sceptre/cli/helpers.py index 413101e5d..c5eeb6dee 100644 --- a/sceptre/cli/helpers.py +++ b/sceptre/cli/helpers.py @@ -93,11 +93,6 @@ def write( output = _generate_yaml(var) if output_format == "text": output = _generate_text(var) - if not no_colour: - stack_status_colourer = StackStatusColourer() - output = stack_status_colourer.colour(str(output)) - - click.echo(output) if file_path: dir_path = file_path.parent @@ -106,6 +101,14 @@ def write( with open(file_path, "w") as f: f.write(output) + return + + if not no_colour: + stack_status_colourer = StackStatusColourer() + output = stack_status_colourer.colour(str(output)) + + click.echo(output) + def _generate_json(stream): encoder = CustomJsonEncoder(indent=4) From 8a771e66a64c1d3e61be70fa30a76e053a516d06 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 13 May 2023 21:09:40 +1000 Subject: [PATCH 31/33] Fix integration tests --- .../features/generate-template-s3.feature | 6 +++--- .../features/generate-template.feature | 18 +++++++++--------- integration-tests/steps/templates.py | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/integration-tests/features/generate-template-s3.feature b/integration-tests/features/generate-template-s3.feature index e0ef65eb3..25140a40d 100644 --- a/integration-tests/features/generate-template-s3.feature +++ b/integration-tests/features/generate-template-s3.feature @@ -3,15 +3,15 @@ Feature: Generate template s3 Scenario: Generating static templates with S3 template handler Given the template for stack "13/B" is "valid_template.json" - When the user generates the template for stack "13/B" + When the user dumps the template for stack "13/B" Then the output is the same as the contents of "valid_template.json" template Scenario: Render jinja templates with S3 template handler Given the template for stack "13/C" is "jinja/valid_template.j2" - When the user generates the template for stack "13/C" + When the user dumps the template for stack "13/C" Then the output is the same as the contents of "valid_template.json" template Scenario: Render python templates with S3 template handler Given the template for stack "13/D" is "python/valid_template.py" - When the user generates the template for stack "13/D" + When the user dumps the template for stack "13/D" Then the output is the same as the contents of "valid_template.json" template diff --git a/integration-tests/features/generate-template.feature b/integration-tests/features/generate-template.feature index 97f374b8e..560a8b53b 100644 --- a/integration-tests/features/generate-template.feature +++ b/integration-tests/features/generate-template.feature @@ -2,7 +2,7 @@ Feature: Generate template Scenario Outline: Generating static templates Given the template for stack "1/A" is "" - When the user generates the template for stack "1/A" + When the user dumps the template for stack "1/A" Then the output is the same as the contents of "" template Examples: Json, Yaml @@ -20,22 +20,22 @@ Feature: Generate template Scenario: Generate template using a valid python template file that outputs json Given the template for stack "1/A" is "valid_template_json.py" - When the user generates the template for stack "1/A" + When the user dumps the template for stack "1/A" Then the output is the same as the contents returned by "valid_template_json.py" Scenario: Generate template using a valid python template file that outputs json with ignore dependencies Given the template for stack "1/A" is "valid_template_json.py" - When the user generates the template for stack "1/A" with ignore dependencies + When the user dumps the template for stack "1/A" with ignore dependencies Then the output is the same as the contents returned by "valid_template_json.py" Scenario: Generate template using a valid python template file that outputs yaml Given the template for stack "1/A" is "valid_template_yaml.py" - When the user generates the template for stack "1/A" + When the user dumps the template for stack "1/A" Then the output is the same as the contents returned by "valid_template_yaml.py" Scenario Outline: Generating erroneous python templates Given the template for stack "1/A" is "" - When the user generates the template for stack "1/A" + When the user dumps the template for stack "1/A" Then a "" is raised Examples: Template Errors @@ -45,12 +45,12 @@ Feature: Generate template Scenario: Generate template using a template file with an unsupported extension Given the template for stack "1/A" is "template.unsupported" - When the user generates the template for stack "1/A" + When the user dumps the template for stack "1/A" Then a "UnsupportedTemplateFileTypeError" is raised Scenario Outline: Rendering jinja templates Given the template for stack "7/A" is "" - When the user generates the template for stack "7/A" + When the user dumps the template for stack "7/A" Then the output is the same as the contents of "" template Examples: Template file extensions @@ -59,7 +59,7 @@ Feature: Generate template Scenario Outline: Render jinja template which uses an invalid key Given the template for stack "7/A" is "" - When the user generates the template for stack "7/A" + When the user dumps the template for stack "7/A" Then a "" is raised Examples: Render Errors @@ -69,5 +69,5 @@ Feature: Generate template Scenario: Generating static templates with file template handler Given the template for stack "13/A" is "valid_template.json" - When the user generates the template for stack "13/A" + When the user dumps the template for stack "13/A" Then the output is the same as the contents of "valid_template.json" template diff --git a/integration-tests/steps/templates.py b/integration-tests/steps/templates.py index dfa49c9de..00ca37a10 100644 --- a/integration-tests/steps/templates.py +++ b/integration-tests/steps/templates.py @@ -104,12 +104,13 @@ def step_impl(context, stack_name): try: context.output = sceptre_plan.dump_template() + print(f"Got output {context.output}") except Exception as e: context.error = e @when( - 'the user generates the template for stack "{stack_name}" with ignore dependencies' + 'the user dumps the template for stack "{stack_name}" with ignore dependencies' ) def step_impl(context, stack_name): sceptre_context = SceptreContext( From 08862a8b6c76ab648207b9e437ee632ce9271d67 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sat, 13 May 2023 21:20:28 +1000 Subject: [PATCH 32/33] More renaming for dump name --- ...erate-template-s3.feature => dump-template-s3.feature} | 4 ++-- .../{generate-template.feature => dump-template.feature} | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) rename integration-tests/features/{generate-template-s3.feature => dump-template-s3.feature} (88%) rename integration-tests/features/{generate-template.feature => dump-template.feature} (94%) diff --git a/integration-tests/features/generate-template-s3.feature b/integration-tests/features/dump-template-s3.feature similarity index 88% rename from integration-tests/features/generate-template-s3.feature rename to integration-tests/features/dump-template-s3.feature index 25140a40d..1684c3d2b 100644 --- a/integration-tests/features/generate-template-s3.feature +++ b/integration-tests/features/dump-template-s3.feature @@ -1,7 +1,7 @@ @s3-template-handler -Feature: Generate template s3 +Feature: Dump template s3 - Scenario: Generating static templates with S3 template handler + Scenario: Dumping static templates with S3 template handler Given the template for stack "13/B" is "valid_template.json" When the user dumps the template for stack "13/B" Then the output is the same as the contents of "valid_template.json" template diff --git a/integration-tests/features/generate-template.feature b/integration-tests/features/dump-template.feature similarity index 94% rename from integration-tests/features/generate-template.feature rename to integration-tests/features/dump-template.feature index 560a8b53b..4e7f4cd89 100644 --- a/integration-tests/features/generate-template.feature +++ b/integration-tests/features/dump-template.feature @@ -1,6 +1,6 @@ -Feature: Generate template +Feature: Dump template - Scenario Outline: Generating static templates + Scenario Outline: Dumping static templates Given the template for stack "1/A" is "" When the user dumps the template for stack "1/A" Then the output is the same as the contents of "" template @@ -33,7 +33,7 @@ Feature: Generate template When the user dumps the template for stack "1/A" Then the output is the same as the contents returned by "valid_template_yaml.py" - Scenario Outline: Generating erroneous python templates + Scenario Outline: Dumping erroneous python templates Given the template for stack "1/A" is "" When the user dumps the template for stack "1/A" Then a "" is raised @@ -67,7 +67,7 @@ Feature: Generate template | jinja/invalid_template_missing_key.j2 | UndefinedError | | jinja/invalid_template_missing_attr.j2 | UndefinedError | - Scenario: Generating static templates with file template handler + Scenario: Dumping static templates with file template handler Given the template for stack "13/A" is "valid_template.json" When the user dumps the template for stack "13/A" Then the output is the same as the contents of "valid_template.json" template From f51e267a00ba9e21b0085b1df1e9f3a75d1ce7f9 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sun, 14 May 2023 11:36:46 +1000 Subject: [PATCH 33/33] Linter --- integration-tests/steps/templates.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration-tests/steps/templates.py b/integration-tests/steps/templates.py index 00ca37a10..b4e66c378 100644 --- a/integration-tests/steps/templates.py +++ b/integration-tests/steps/templates.py @@ -109,9 +109,7 @@ def step_impl(context, stack_name): context.error = e -@when( - 'the user dumps the template for stack "{stack_name}" with ignore dependencies' -) +@when('the user dumps the template for stack "{stack_name}" with ignore dependencies') def step_impl(context, stack_name): sceptre_context = SceptreContext( command_path=stack_name + ".yaml",