diff --git a/.github/workflows/branch-integration-tests.yaml b/.github/workflows/branch-integration-tests.yaml new file mode 100644 index 000000000..fb73e15e3 --- /dev/null +++ b/.github/workflows/branch-integration-tests.yaml @@ -0,0 +1,19 @@ +# The idea with this workflow is to allow core reviewers to trigger the +# integration tests by pushing a branch to the sceptre repository. +name: branch-integration-tests + +on: + push: + branches: + - '*' # matches every branch that doesn't contain a '/' + - '*/*' # matches every branch containing a single '/' + - '**' # matches every branch + - '!master' # excludes master + +jobs: + integration-tests: + if: ${{ github.ref != 'refs/heads/master' }} + uses: "./.github/workflows/integration-tests.yaml" + with: + # role generated from https://github.com/Sceptre/sceptre-aws/blob/master/config/prod/gh-oidc-sceptre-tests.yaml + role-to-assume: "arn:aws:iam::743644221192:role/gh-oidc-sceptre-tests" diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index d037d1cac..c6e711d47 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -1,3 +1,4 @@ +# Execute sanity checks name: check on: diff --git a/.github/workflows/comment-integration-tests.yaml b/.github/workflows/comment-integration-tests.yaml new file mode 100644 index 000000000..a05829a8f --- /dev/null +++ b/.github/workflows/comment-integration-tests.yaml @@ -0,0 +1,18 @@ +# The idea with this workflow is to allow users to trigger an integration test +# run from a PR however it doesn't work because github action does not allow +# access to the github token when triggered from a PR. The workflow fails with.. +# "Credentials could not be loaded, please check your action inputs: Could not load credentials from any providers" + +name: comment-integration-tests + +on: + pull_request_review: + types: [submitted] + +jobs: + integration-tests: + if: ${{ contains(github.event.review.body, '/integration-tests') }} + uses: "./.github/workflows/integration-tests.yaml" + with: + # role generated from https://github.com/Sceptre/sceptre-aws/blob/master/config/prod/gh-oidc-sceptre-tests.yaml + role-to-assume: "arn:aws:iam::743644221192:role/gh-oidc-sceptre-tests" diff --git a/.github/workflows/gate.yaml b/.github/workflows/gate.yaml index be6d21304..ccbb0ed17 100644 --- a/.github/workflows/gate.yaml +++ b/.github/workflows/gate.yaml @@ -1,11 +1,7 @@ +# Run integration tests when a PR is merged to master and publish a +# docker container (with an `edge` tag) with the latest code name: gate -env: - AWS_REGION: us-east-1 - AWS_ROLE_DURATION: 3600 - # role generated from https://github.com/Sceptre/sceptre-aws/blob/master/config/prod/gh-oidc-sceptre-tests.yaml - AWS_ROLE: arn:aws:iam::743644221192:role/gh-oidc-sceptre-tests - on: workflow_run: workflows: @@ -18,38 +14,10 @@ on: jobs: integration-tests: if: ${{ github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - permissions: - id-token: write - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - steps: - - uses: actions/checkout@v4 - - name: Install Poetry - uses: snok/install-poetry@v1 - - name: Install dependencies - run: poetry install --no-interaction --all-extras - # Update poetry for https://github.com/python-poetry/poetry/issues/7184 - - name: update poetry - run: poetry self update --no-ansi - - name: Setup Python - id: setup-python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - cache: 'poetry' - - name: Assume AWS role - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ env.AWS_ROLE }} - role-session-name: GHA-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }} - role-duration-seconds: ${{ env.AWS_ROLE_DURATION }} - - name: run tests - run: poetry run behave integration-tests/features --junit --junit-directory build/behave - env: - AWS_DEFAULT_REGION: eu-west-1 + uses: "./.github/workflows/integration-tests.yaml" + with: + # role generated from https://github.com/Sceptre/sceptre-aws/blob/master/config/prod/gh-oidc-sceptre-tests.yaml + role-to-assume: "arn:aws:iam::743644221192:role/gh-oidc-sceptre-tests" docker-build-push: needs: diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml new file mode 100644 index 000000000..86a6368e2 --- /dev/null +++ b/.github/workflows/integration-tests.yaml @@ -0,0 +1,55 @@ +name: integration-tests + +on: + workflow_call: + inputs: + aws-region: + type: string + default: us-east-1 + role-to-assume: + required: true + type: string + role-duration-seconds: + type: number + default: 3600 + +jobs: + tests: + runs-on: ubuntu-latest + permissions: + id-token: write + # There is only one AWS account for running integration tests and the tests are not designed + # to run concurrently in one account which is why we are disabling concurrency. + # The intention is to have all triggered integration tests execute serially in one queue, + # all triggered integration tests should wait in the queue however github is canceling + # waiting jobs in the queue. Github currently does not support the desired use case, + # more info at https://github.com/orgs/community/discussions/41518 + concurrency: + group: integration-tests + cancel-in-progress: false + steps: + - uses: actions/checkout@v4 + - name: Install Poetry + uses: snok/install-poetry@v1 + - name: Install dependencies + run: poetry install --no-interaction --all-extras + # Update poetry for https://github.com/python-poetry/poetry/issues/7184 + - name: update poetry + run: poetry self update --no-ansi + - name: Setup Python + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + - name: Assume AWS role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws-region }} + role-to-assume: ${{ inputs.role-to-assume }} + role-session-name: GHA-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }} + role-duration-seconds: ${{ inputs.role-duration-seconds }} + - name: run tests + run: poetry run behave integration-tests/features --junit --junit-directory build/behave + env: + AWS_DEFAULT_REGION: eu-west-1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 084a7c93a..9c25517c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v0.83.8 + rev: v0.85.0 hooks: - id: cfn-python-lint args: @@ -36,7 +36,7 @@ repos: ^.pre-commit-config.yaml ) - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 24.1.1 hooks: - id: black - repo: https://github.com/python-poetry/poetry diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3cd59876..08fb35b25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,7 +85,7 @@ $ git clone git@github.org:/sceptre.git [virtual environment](http://docs.python-guide.org/en/latest/dev/virtualenvs/)) ```bash - $ poetry install --all-extras -v +$ poetry install --all-extras -v ``` 4. Create a branch for local development: diff --git a/README.md b/README.md index 79cddc620..f47630a17 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Sceptre -[![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/Sceptre/sceptre/main.yaml)](https://github.com/Sceptre/sceptre/actions/workflows/main.yaml) +[![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/Sceptre/sceptre/gate.yaml)](https://github.com/Sceptre/sceptre/actions/workflows/gate.yaml) [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/sceptreorg/sceptre?logo=docker&sort=semver)](https://hub.docker.com/r/sceptreorg/sceptre) [![PyPI](https://img.shields.io/pypi/v/sceptre?logo=pypi)](https://pypi.org/project/sceptre/) [![PyPI - Status](https://img.shields.io/pypi/status/sceptre?logo=pypi)](https://pypi.org/project/sceptre/) diff --git a/pyproject.toml b/pyproject.toml index 94bac8abd..91d78e158 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,11 +15,6 @@ classifiers = [ "Intended Audience :: Developers", "Natural Language :: English", "Environment :: Console", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", ] [tool.poetry.scripts] diff --git a/sceptre/connection_manager.py b/sceptre/connection_manager.py index 0e752c946..a27ad3f4b 100644 --- a/sceptre/connection_manager.py +++ b/sceptre/connection_manager.py @@ -316,9 +316,9 @@ def _get_session( "RoleSessionName": session_name, } if self.sceptre_role_session_duration: - assume_role_kwargs[ - "DurationSeconds" - ] = self.sceptre_role_session_duration + assume_role_kwargs["DurationSeconds"] = ( + self.sceptre_role_session_duration + ) sts_response = sts_client.assume_role(**assume_role_kwargs) credentials = sts_response["Credentials"] diff --git a/sceptre/diffing/stack_differ.py b/sceptre/diffing/stack_differ.py index 87ad33e79..6f1603ec6 100644 --- a/sceptre/diffing/stack_differ.py +++ b/sceptre/diffing/stack_differ.py @@ -19,6 +19,7 @@ from cfn_tools import ODict from yaml import Dumper +from sceptre.exceptions import SceptreException from sceptre.plan.actions import StackActions from sceptre.stack import Stack @@ -173,9 +174,18 @@ def _extract_parameters_from_generated_stack(self, stack: Stack) -> dict: if value is None: continue - if isinstance(value, list): - value = ",".join(item.rstrip("\n") for item in value) - formatted_parameters[key] = value.rstrip("\n") + try: + if isinstance(value, list): + value = ",".join(item.rstrip("\n") for item in value) + formatted_parameters[key] = value.rstrip("\n") + # Other unexpected data can get through and this would blow up the differ + # and lead to quite confusing exceptions being raised. This check here could + # be removed in a future version of Sceptre if the reader class did sanity checking. + except AttributeError: + raise SceptreException( + f"Parameter '{key}' whose value is {value} " + f"is of type {type(value)} and not expected here" + ) return formatted_parameters @@ -361,7 +371,7 @@ def __init__( self, show_no_echo=False, *, - universal_template_loader: Callable[[str], Tuple[dict, str]] = cfn_flip.load + universal_template_loader: Callable[[str], Tuple[dict, str]] = cfn_flip.load, ): """Initializes a DeepDiffStackDiffer. @@ -409,7 +419,7 @@ def __init__( self, show_no_echo=False, *, - universal_template_loader: Callable[[str], Tuple[dict, str]] = cfn_flip.load + universal_template_loader: Callable[[str], Tuple[dict, str]] = cfn_flip.load, ): """Initializes a DifflibStackDiffer. diff --git a/sceptre/stack.py b/sceptre/stack.py index 35c1cade3..8a633096c 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -42,7 +42,7 @@ class Stack: :param template_path: The relative path to the CloudFormation, Jinja2, or Python template to build the Stack from. If this is filled, `template_handler_config` should not be filled. This field has been deprecated since - version 4.0.0 and will be removed in version 5.0.0. + version 4.0.0 and will be removed eventually. :param template_handler_config: Configuration for a Template Handler that can resolve its arguments to a template string. Should contain the `type` property to specify @@ -92,15 +92,15 @@ class Stack: :param iam_role: The ARN of a role for Sceptre to assume before interacting with the environment. If not supplied, Sceptre uses the user's AWS CLI - credentials. This field has been deprecated since version 4.0.0 and will be removed in - version 5.0.0. + credentials. This field has been deprecated since version 4.0.0 and will be removed + eventually. :param sceptre_role: The ARN of a role for Sceptre to assume before interacting\ with the environment. If not supplied, Sceptre uses the user's AWS CLI\ credentials. :param iam_role_session_duration: The duration in seconds of the assumed IAM role session. - This field has been deprecated since version 4.0.0 and will be removed in version 5.0.0. + This field has been deprecated since version 4.0.0 and will be removed eventually. :param sceptre_role_session_duration: The duration in seconds of the assumed IAM role session. @@ -154,14 +154,23 @@ class Stack: hooks = HookProperty("hooks") iam_role = create_deprecated_alias_property( - "iam_role", "sceptre_role", "4.0.0", "5.0.0" + "iam_role", + "sceptre_role", + deprecated_in="4.0.0", + removed_in=None, ) role_arn = create_deprecated_alias_property( - "role_arn", "cloudformation_service_role", "4.0.0", "5.0.0" + "role_arn", + "cloudformation_service_role", + deprecated_in="4.0.0", + removed_in=None, ) sceptre_role_session_duration = None iam_role_session_duration = create_deprecated_alias_property( - "iam_role_session_duration", "sceptre_role_session_duration", "4.0.0", "5.0.0" + "iam_role_session_duration", + "sceptre_role_session_duration", + deprecated_in="4.0.0", + removed_in=None, ) def __init__( @@ -388,7 +397,10 @@ def template(self): @property @deprecated( - "4.0.0", "5.0.0", __version__, "Use the template Stack Config key instead." + deprecated_in="4.0.0", + removed_in=None, + current_version=__version__, + details="Use the template Stack Config key instead.", ) def template_path(self) -> str: """The path argument from the template_handler config. This field is deprecated as of v4.0.0 @@ -398,7 +410,10 @@ def template_path(self) -> str: @template_path.setter @deprecated( - "4.0.0", "5.0.0", __version__, "Use the template Stack Config key instead." + deprecated_in="4.0.0", + removed_in=None, + current_version=__version__, + details="Use the template Stack Config key instead.", ) def template_path(self, value: str): self.template_handler_config = {"type": "file", "path": value} diff --git a/tests/test_diffing/test_stack_differ.py b/tests/test_diffing/test_stack_differ.py index b55bbb876..83bf7b441 100644 --- a/tests/test_diffing/test_stack_differ.py +++ b/tests/test_diffing/test_stack_differ.py @@ -16,6 +16,7 @@ DeepDiffStackDiffer, DifflibStackDiffer, ) +from sceptre.exceptions import SceptreException from sceptre.plan.actions import StackActions from sceptre.stack import Stack @@ -343,6 +344,17 @@ def test_diff__generated_stack_has_none_for_parameter_value__its_treated_like_it self.expected_deployed_config, expected_generated ) + def test_diff__generated_stack_has_a_bool( + self, + ): + self.parameters_on_stack_config["new"] = True + message = ( + "Parameter 'new' whose value is True is of type " + " and not expected here" + ) + with pytest.raises(SceptreException, match=message): + self.differ.diff(self.actions) + def test_diff__stack_exists_with_same_config_but_template_does_not__compares_identical_configs( self, ): @@ -421,9 +433,9 @@ def test_diff__generated_template_has_no_echo_parameter__masks_value(self): self.local_no_echo_parameters.append("hide_me") expected_generated_config = self.expected_generated_config - expected_generated_config.parameters[ - "hide_me" - ] = StackDiffer.NO_ECHO_REPLACEMENT + expected_generated_config.parameters["hide_me"] = ( + StackDiffer.NO_ECHO_REPLACEMENT + ) self.differ.diff(self.actions)