diff --git a/.circleci/README.md b/.circleci/README.md index bb1e4ffaa..9c4de7cd0 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -2,14 +2,30 @@ * `REPOSITORY_PATH` - directory in container where dest repo is initialized within container * `DEPLOYMENT_GIT_SSH` or `DEPLOYMENT_GIT_HTTPS`' - destination repository for rendered pages - * `DEPLOYMENT_GIT_SSH` - requires adding SSH key to the CircleCI + deploy key with push rights to the target repository + * `DEPLOYMENT_GIT_SSH` - requires adding SSH key to the CircleCI + deploy key with push rights to the target repository * `DEPLOYMENT_GIT_HTTPS` - requires setting up personal access token with push rights to the target repository * `GITHUB_EMAIL` - email which is associated with commit message and github account * `GITHUB_TOKEN` - access token with push rights to the target repository `/docs` * `CIRCLE_USERNAME` - built in variable in CIRCLE CI - should be same as user who pushes to repository +* `CI_SERVICE_AWS_ACCESS_KEY_ID` - The CI service user's AWS access key ID +* `CI_SERVICE_AWS_SECRET_ACCESS_KEY` - The CI service user's AWS secret access key ##### Optional evn variables * `RENDERED_DOCS_DIR` - directory where rendered pages are stored within `DEST_REPO` - default is repository root * `DOCS_DEV_VERSION` - directory name where latest version of docs are published, - default value is `dev` will result in path `../docs/dev` \ No newline at end of file + default value is `dev` will result in path `../docs/dev` + + + +##### Release Workflow + +This is the workflow used to publish Sceptre releases. + +* Verify that the latest CI build for Sceptre is green on the `master` branch +* Checkout the latest master branch +* Bump the Sceptre version (i.e. 2.4.0 -> 2.5.0) & update the CHANGELOG.md file +* Commit and push the change +* Create a tag (i.e. tag -a -s v2.6.0) +* Push tag +* The `publish` workflow is triggered on CI which will publish a new Sceptre release. diff --git a/.circleci/add-known-hosts.sh b/.circleci/add-known-hosts.sh index 984e3177d..a6efa983c 100644 --- a/.circleci/add-known-hosts.sh +++ b/.circleci/add-known-hosts.sh @@ -11,4 +11,4 @@ fi mkdir -p ~/.ssh -ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts \ No newline at end of file +ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bce30650..8217d529e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,16 +1,19 @@ version: 2.1 +orbs: + twitter-orb: jakousa/twitter-orb@1.0.1 + executors: docker-publisher: environment: - IMAGE_NAME: cloudreach/sceptre + IMAGE_NAME: sceptreorg/sceptre docker: - image: circleci/buildpack-deps:stretch aliases: - &docs-job docker: - - image: cloudreach/sceptre-circleci:0.4 + - image: sceptreorg/sceptre-circleci:2.0.0 environment: REPOSITORY_PATH: '/home/circleci/docs' DEPLOYMENT_GIT_SSH: 'git@github.com:Sceptre/sceptre.github.io.git' @@ -25,130 +28,103 @@ aliases: command: | chmod +x .circleci/add-known-hosts.sh ./.circleci/add-known-hosts.sh - . ./venv/bin/activate + pyenv global venv chmod +x .circleci/github-pages.sh ./.circleci/github-pages.sh jobs: build: docker: - - image: cloudreach/sceptre-circleci:0.4 + - image: sceptreorg/sceptre-circleci:2.0.0 steps: - checkout - run: name: 'Creating Virtualenv' - command: virtualenv venv + command: | + pyenv virtualenv 3.9.4 venv - restore_cache: key: - sceptre-{{ .Environment.CACHE_VERSION }}-requirements-{{ arch }}-{{ + sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{ checksum "requirements/prod.txt" }}-{{ checksum "requirements/dev.txt" }} - run: - name: 'Installing Requirements' + name: 'Installing Dependencies' command: | - . ./venv/bin/activate - pip install -r requirements/prod.txt - pip install -r requirements/dev.txt - pip install awscli + pyenv global venv + make install-dev - save_cache: key: - sceptre-{{ .Environment.CACHE_VERSION }}-requirements-{{ arch }}-{{ + sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{ checksum "requirements/prod.txt" }}-{{ checksum "requirements/dev.txt" }} paths: - - venv + - ../.pyenv/versions/3.9.4/envs/venv - run: name: 'Installing Sceptre' command: | - . ./venv/bin/activate + pyenv global venv pip install . - persist_to_workspace: root: /home/circleci paths: - project + - .pyenv lint-and-unit-tests: docker: - - image: cloudreach/sceptre-circleci:0.4 + - image: sceptreorg/sceptre-circleci:2.0.0 steps: - attach_workspace: at: /home/circleci - restore_cache: - key: - sceptre-{{ .Environment.CACHE_VERSION }}-tox-requirements-{{ arch - }}-{{ checksum "requirements/prod.txt" }}-{{ checksum - "requirements/dev.txt" }} -{{ checksum "tox.ini" }} + keys: + - sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{ + checksum "requirements/prod.txt" }}-{{ checksum "requirements/dev.txt" }} + - sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{ + checksum "requirements/prod.txt" }}-{{ checksum + "requirements/dev.txt" }}-{{ checksum "tox.ini" }} - run: name: 'Linting' command: | - . ./venv/bin/activate + pyenv global venv make lint - - run: name: 'Unit Test' command: | + pyenv global venv make test-all - - - store_test_results: - path: test-results - destination: test-results - - - store_artifacts: - path: test-results - destination: test-results - - - run: - name: 'Coverage' - command: | - . venv/bin/activate - make coverage - - - store_test_results: - path: coverage.xml - destination: coverage-reports - - - store_artifacts: - path: coverage.xml - destination: coverage-reports - - - persist_to_workspace: - root: /home/circleci - paths: - - project - - save_cache: key: - sceptre-{{ .Environment.CACHE_VERSION }}-tox-requirements-{{ arch + sceptre-{{ .Environment.CACHE_VERSION }}-{{ arch }}-{{ checksum "requirements/prod.txt" }}-{{ checksum "requirements/dev.txt" }}-{{ checksum "tox.ini" }} paths: - .tox - - sonar: - docker: - - image: cloudreach/sceptre-circleci-sonarqube:latest - steps: - - attach_workspace: - at: /home/circleci - - run: - name: Run Sonarqube - command: | - . venv/bin/activate - make sonar + - store_test_results: + path: coverage.xml + destination: coverage-reports integration-tests: parallelism: 2 docker: - - image: cloudreach/sceptre-circleci:0.4 + - image: sceptreorg/sceptre-circleci:2.0.0 environment: AWS_DEFAULT_REGION: eu-west-1 steps: - attach_workspace: at: /home/circleci + - restore_cache: + key: + sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{ + checksum "requirements/prod.txt" }}-{{ checksum + "requirements/dev.txt" }} - run: name: 'Integration Testing' command: | - . ./venv/bin/activate + pyenv global venv + mkdir -p ~/.aws + echo -e "[default]\nregion=eu-west-1\nsource_profile=default\nrole_arn=arn:aws:iam::743644221192:role/sceptre-integration-test-ServiceRole-1SHK9LY0T6P3F" > ~/.aws/config + echo -e "[default]\nregion=eu-west-1\naws_access_key_id=$CI_SERVICE_AWS_ACCESS_KEY_ID\naws_secret_access_key=$CI_SERVICE_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials behave --junit \ --junit-directory build/behave \ $(circleci tests glob "integration-tests/features/*.feature" | circleci tests split --split-by=timings) @@ -182,20 +158,29 @@ jobs: deploy-pypi: docker: - - image: cloudreach/sceptre-circleci:0.4 + - image: sceptreorg/sceptre-circleci:2.0.0 steps: - attach_workspace: at: /home/circleci + - restore_cache: + key: + sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{ + checksum "requirements/prod.txt" }}-{{ checksum + "requirements/dev.txt" }} + - run: + name: 'Installing Dependencies' + command: | + pyenv global venv + make install-dev - run: name: 'Create Distributions' command: | - . ./venv/bin/activate + pyenv global venv make dist - run: name: 'Upload Distributions' command: | - . ./venv/bin/activate - pip install twine + pyenv global venv twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD dist/* deploy-latest-dockerhub: @@ -231,39 +216,59 @@ jobs: docker push $IMAGE_NAME:latest docker push $IMAGE_NAME:$IMAGE_TAG + tweet-release: + docker: + - image: jakousa/twurl:latest + resource_class: small + steps: + - twitter-orb/tweet: + access_secret: TOKEN_SECRET + access_token: ACCESS_TOKEN + consumer_key: API_KEY + consumer_secret: API_SECRET + contents: | + (echo $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME) has released << pipeline.git.tag >>: + << pipeline.project.git_url >>/releases/tag/<< pipeline.git.tag >> + workflows: - version: 2.1 - build-test-and-deploy: + build-and-unit-test: jobs: - build: - context: sceptre-core filters: - tags: - only: /.*/ - + branches: + only: /^pull\/.*/ - lint-and-unit-tests: - context: sceptre-core requires: - build filters: - tags: - only: /.*/ - - sonar: - context: sceptre-core + branches: + only: /^pull\/.*/ + - build-docker-image: requires: - - build - lint-and-unit-tests + filters: + branches: + only: /^pull\/.*/ + build-test-and-deploy: + jobs: + - build: + filters: + branches: + ignore: /^pull\/.*/ + - lint-and-unit-tests: + requires: + - build + filters: + branches: + ignore: /^pull\/.*/ - integration-tests: context: sceptre-core requires: - build filters: - tags: - only: /.*/ branches: ignore: /^pull\/.*/ - - deploy-docs-branch: context: sceptre-core requires: @@ -271,67 +276,71 @@ workflows: - integration-tests filters: branches: - only: master - - - deploy-docs-tag: - context: sceptre-core + ignore: /^pull\/.*/ + - build-docker-image: requires: - lint-and-unit-tests - integration-tests + filters: + branches: + ignore: /^pull\/.*/ + - deploy-latest-dockerhub: + context: sceptreorg-dockerhub + requires: + - build-docker-image + filters: + branches: + ignore: /^pull\/.*/ + + publish: + jobs: + - build: filters: tags: - only: /^v.*/ + only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ branches: ignore: /.*/ - - - deploy-pypi-approval: + - deploy-pypi: context: sceptre-core - type: approval requires: - - lint-and-unit-tests - - integration-tests - - sonar + - build filters: tags: - only: /^v[0-9]+(\.[0-9]+)*/ + only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ branches: ignore: /.*/ - - - deploy-pypi: + - deploy-docs-tag: context: sceptre-core requires: - - deploy-pypi-approval + - deploy-pypi filters: tags: - only: /^v[0-9]+(\.[0-9]+)*/ + only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ branches: ignore: /.*/ - - build-docker-image: - context: sceptre-core requires: - - lint-and-unit-tests - - integration-tests - - sonar + - deploy-pypi filters: tags: - only: /.*/ - - - deploy-latest-dockerhub: - context: sceptre-core + only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ + branches: + ignore: /.*/ + - deploy-dockerhub-tagged: + context: sceptreorg-dockerhub requires: - build-docker-image filters: + tags: + only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ branches: - only: master - - - deploy-dockerhub-tagged: - context: sceptre-core + ignore: /.*/ + - tweet-release: + context: sceptreorg-twitter requires: - - build-docker-image - - deploy-pypi + - deploy-dockerhub-tagged filters: tags: - only: /^v.*/ + only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ branches: ignore: /.*/ diff --git a/.circleci/documentation-versions.py b/.circleci/documentation-versions.py index 73db5987e..d0a7378c8 100644 --- a/.circleci/documentation-versions.py +++ b/.circleci/documentation-versions.py @@ -52,13 +52,13 @@ def main(): if item.is_dir() and VERSION_REGEX.match(item.name) ), reverse=True, - key=attrgetter("name") + key=attrgetter("name"), ) active_versions = ( - ["latest", "dev"] - + [item.name for item in documentation_directories[:NUMBER_OF_VERSIONS_TO_KEEP]] - + KEEP_VERSIONS + ["latest", "dev"] + + [item.name for item in documentation_directories[:NUMBER_OF_VERSIONS_TO_KEEP]] + + KEEP_VERSIONS ) versions_to_remove = ( item.path diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..d1b533d04 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space + +[*.json] +indent_size = 2 + +[*.{markdown,md}] +indent_size = 4 +max_line_length = 80 +trim_trailing_whitespace = false + +[*.py] +indent_size = 4 +max_line_legth = 120 + +[*.{yaml,yml}] +indent_size = 2 + +[Makefile] +indent_style = tab diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..c1f90c901 --- /dev/null +++ b/.flake8 @@ -0,0 +1,16 @@ +[flake8] +exclude = + .git, + __pycache__, + build, + dist, + .tox, + venv, + .venv, + .pytest_cache +max-complexity = 12 +per-file-ignores = + docs/_api/conf.py: E265 + integration-tests/steps/*: E501,F811,F403,F405 +extend-ignore = E203 +max-line-length = 120 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..0f9c40e51 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,17 @@ +### Subject of the issue +Describe your issue here. + +### Your environment +* version of sceptre (sceptre --version) +* version of python (python --version) +* which OS/distro + +### Steps to reproduce +Tell us how to reproduce this issue. Please provide sceptre projct files if possible, +you can use https://plnkr.co/edit/ANFHm61Ilt4mQVgF as a base. + +### Expected behaviour +Tell us what should happen + +### Actual behaviour +Tell us what happens instead diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d16a0eef7..91b414eeb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,7 +8,7 @@ - [ ] Added/Updated integration tests (if applicable). - [ ] All unit tests (`make test`) are passing. - [ ] Used the same coding conventions as the rest of the project. -- [ ] The new code passes flake8 (`make lint`) checks. +- [ ] The new code passes pre-commit validations (`pre-commit run --all-files`). - [ ] The PR relates to _only_ one subject with a clear title. and description in grammatically correct, complete sentences. diff --git a/.gitignore b/.gitignore index c38aba7f7..994c00d19 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ __pycache__/ .Python env/ venv/ +.venv/ build/ develop-eggs/ dist/ @@ -24,6 +25,7 @@ var/ *.egg-info/ .installed.cfg *.egg +.python-version # PyInstaller # Usually these files are written by a python script from a template @@ -138,3 +140,6 @@ Session.vim tags # Persistent undo [._]*.un~ + +# temp files +temp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..0d4acd697 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +default_language_version: + python: python3 + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + - repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + - repo: https://github.com/adrienverge/yamllint + rev: v1.29.0 + hooks: + - id: yamllint + - repo: https://github.com/awslabs/cfn-python-lint + rev: v0.72.10 + hooks: + - id: cfn-python-lint + args: + - "-i=E0000" + - "-i=E1001" + - "-i=E3012" + - "-i=W6001" + exclude: | + (?x)( + ^integration-tests/sceptre-project/config/| + ^integration-tests/sceptre-project/templates/jinja/| + ^tests/fixtures-vpc/config/| + ^tests/fixtures/config/| + ^temp/| + ^.circleci/| + ^.pre-commit-config.yaml + ) + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version + language_version: python3.10 + - repo: https://github.com/AleksaC/circleci-cli-py + rev: v0.1.25638 + hooks: + - id: circle-ci-validator diff --git a/.python-version b/.python-version deleted file mode 100644 index 8b3ce08f7..000000000 --- a/.python-version +++ /dev/null @@ -1,3 +0,0 @@ -2.7-dev -3.6-dev -3.7-dev diff --git a/.yamllint b/.yamllint new file mode 100644 index 000000000..b83bc42d6 --- /dev/null +++ b/.yamllint @@ -0,0 +1,27 @@ +--- + +extends: default + +rules: + braces: + level: warning + max-spaces-inside: 1 + brackets: + level: warning + max-spaces-inside: 1 + commas: + level: warning + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: + level: warning + hyphens: + level: warning + indentation: + level: warning + indent-sequences: consistent + line-length: disable + truthy: disable + new-line-at-end-of-file: + level: warning diff --git a/CHANGELOG.md b/CHANGELOG.md index 04bc9605e..8fce4c81c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,327 @@ Categories: Added, Removed, Changed, Fixed, Nonfunctional, Deprecated ## Unreleased +## 4.0.2 (2023.02.20) +### Fixed + - [Resolve #1307] Fixing Connection Manager bug (#1308) + +## 4.0.1 (2023.02.12) + +### Changed + - Supporting `!rcmd` resolver up to v2 + +## 4.0.0 (2023.02.08) + +### Added + - [Resolve #1283] Introducing `sceptre_role`, `cloudformation_service_role` (#1295) + - These are just iam_role and role_arn renamed to be a lot clearer. See "Deprecations" below. + + +### Changed + - [Resolve #1299] Making the ConnectionManager a more "friendly" interface for hooks, resolvers, + and template handlers (#1287, #1300) + - This creates adds the public `get_session()` and + `create_session_environment_variables()` methods to make AWS interactions + easier and more consistent with individual stack configurations for + iam_role, profile, and region configurations. + - The `call()` method now properly distinguishes between default stack + configurations for profile, region, and `sceptre_role` and setting those to + `None` to nullify them. + - Preventing Duplicate Deprecation Warnings from being emitted (#1297) + +#### _Potentially_ Breaking Changes + - The !cmd hook now invokes the passed command using the AWS environment + variables that correspond with the stack's IAM configurations (i.e. iam_role, + profile, region). This means that the hook will operate the same as every + other part of Sceptre and regard how the stack is configured. This should make + it easier to invoke other tools like AWS CLI with your hooks. However, if + your project is setting environment variables with the intent to change how + the command authenticates with AWS (such as a different token, profile, or + region), these environment variables will be overridden. To maintain the same + functionality, you should prefix your command with + `export AWS_SESSION_TOKEN={{environment_variable.AWS_SESSION_TOKEN}} &&` (or + whatever other environment variable(s) you need to explicitly set). + +### Deprecations + - [Resolve #1283] Deprecating `iam_role`, `role_arn`, and `template_path` (#1295) + - `iam_role` and `role_arn` have been aliased to `sceptre_role` and + `cloudformation_service_role`. Using these fields will result in a + DeprecationWarning. + - `template_path` has actually been slated for removal since v2.7. `template` + should be used instead. Using `template_path` will result in a + DeprecationWarning. + - All three deprecated StackConfig fields will be removed in v5.0.0. + +## 3.3.0 (2023.02.06) + +### Added + - [Resolve #1261] Add coloured differ (#1260) + - Implements coloured diffs for the diff (difflib) command. Responds to --no-color. + - [Resolves #1271] Extend stack colourer to include "IMPORT" states (#1272) + - [Resolves #1179] cloudformation disable-rollback option (#1282) + - Allow user to disable a cloudformation rollback on a sceptre deployment. + +### Changed + - [Resolve #1098] Deploy docker container to sceptreorg repo (#1265) + - Deploy sceptre docker images to dockerhub sceptreorg repo instead of cloudreach repo + - Updating Setuptools and wheel versions to avert security issues + - [Resolve #1293] Improve the Stack Config Jinja Syntax Error Message to include the Stack Name (#1294) + - [Resolves #1267] Improve the Stack Config Jinja Error Message to include the Stack Name (#1269) + +### Fixed + - [Resolve #1273] Events start from response time (#1275) + - Resolves #1273 by starting event filtering from the timestamp returned in + the AWS response headers rather than relying on the workstation clock. + - [Resolve #1253] Failed downloads raise error (#1277) + - Throwing an informative error when the template fails to download instead + of passing the error message to CloudFormation. + - [Resolves #1179] Changed disable-rollback default to None (#1288) + - We want the default value to be None to represent "Do whatever's configured + in the StackConfig" and True/False will override the StackConfig. + +### Nonfunctional + - Add tweet-release to CircleCI config (#1259) + - [Resolves #1276] Adopt Black Auto-formatter (#1278) + - Reformatting all Python files using the Black code formatter. This also + delivers a new function for generating `__repr__` methods which was needed + to deal with a line-too-long issue in Template. Per discussion in #1276 this + PR also disables E203 in flake8. + - Update sceptre-circleci docker image (#1284) + - Update to build and test with a docker image that's based on the official + circleci python docker image. + - [Resolve #1264] Updating the CDK docs to point to the new sceptre-cdk-handler + (#1285) + - This updates our docs to no longer reference the old CDK approach (which + didn't work with CDK assets). In its place, it references the new + sceptre-cdk-handler package that covers that functionality. + +## 3.2.0 (2022.09.20) + +### Added + - [Resolve #1225] Added Python 3.10 support (#1227) + - Implemented a `sceptre dump config` command (#1220) + - [Resolve #1224] Add --drifted option to `drift show` command (#1246) + - [Resolve #1212] Conditional Stacks via "ignore" and "obsolete"; + `sceptre prune` command for cleaning up obsolete stacks (#1229) + +### Changed + - [Resolve #1225] Updating `troposphere` version for python 3.10 compatibility (#1226) + - [Resolve #1225] Updating Sceptre to use importlib on Python 3.10 (#1240) + +### Fixed + - [Resolve #1218] Fixing false-diff reports when parameters end with linebreaks (#1219) + - [Resolve #1236] Updating `networkx` to miticate CWE-502 (#1243) + - [Resolve #1245] Updating `setuptools` version to address build failure (#1247) + - [Resolve #1234] Bugfix to support empty var files (#1248) + - [Resolve #1223] Fix crash when resolvers return lists of `None` (#1249) + +### Nonfunctional + - Added sponsors section to CONTRIBUTING.md (#1221) + - Made unit tests run in parallel when running `make test-all` (#1231) + - Removing awscli from CircleCI builds (#1232) + - Updated sceptre-circleci docker image for Python 3.10 support (#1230) + - Using CircleCI cache to speed up builds (#1242) + +## 3.1.0 (2022.04.13) + +### Added + - [Resolve #1080] Added duration_seconds parameter to adopt DurationSeconds in boto (#1210) + +### Changed + - [Resolve #1203] Updating packaging requirement (#1211) + +## 3.0.0 (2022.02.22) + +### Breaking Changes + - Python 3.6 support has been removed due to that version reaching end-of-life status + - Jinja2 has been upgraded to v3.0 + +### Added + - [Resolve #1114,#426] Resolvable stack_tags (#1184) + - [Resolve #1114,#886,#491] Resolvable role_arn and template_bucket_name (#1153) + - [Resolve #1114] Resolvable iam_role (#1188) + - [Resolve #1114] Resolvable Template Handler configs and the !stack_attr resolver + - [Resolve #1167] Add list stacks command (#1168) + - [Resolve #1169] Add drift detect and drift show commands (#1170) + +### Removed + - [Resolves #1201] Remove Py3.6 support (#1206) + +### Changed + - [Resolve #1114,#1000] Placeholders for non-deployed stacks in non-deployment commands (#1185) + +### Fixed + - [Resolves #1201] Fix dependency conflicts (Jinja2, moto) (#1206) + +### Nonfunctional +- [Resolves #1194] Docs: "know"->"knows" (#1195) +- docs: fix template path in getting-started (#1198) +- Fix spelling of stack_group_config (#1199) + +## 2.7.1 (2021.12.06) + +### Fixed + +- [Resolve #1175] Adding commas for cfn-flip dependency (#1176) +- [Partially resolves #1174] Fixing Docs deployment by pinning Sphinx to lesser version (#1171) + +### Nonfunctional + +- Fix typo in CDK doc (#1181) +- Add release instructions (#1162) +- Resolve #1163 update doc link to new domain (#1166) +- Pointing SAM docs toward the sceptre-sam-handler (#1164) + +## 2.7.0 (2021.11.18) + +### Added + +- [Resolve #966] Add support for J2 Environment configuration +- [Resolves #919] Add merge_keys option (#928) +- [Resolves #1064] Add feature list change-set --url (#1065) +- [Resolves #213] Add support for template handlers (#1088) +- [Resolves #1106] S3 template handler jinja and python support (#1110) +- [Resolves #1124] http template handler (#1125) +- Set file as the default template handler type (#1127) +- Add retry and timeout to http template handler (#1145) +- [Resolve # 683] Introducing the Diff Command (#1132) + +### Fixed + +- [Resolves #813] Fix recursive config render (#1083) +- [Resolve #1096] Gracefully executing SAM Change sets (#1099) +- [Resolves 556] fix incorrect stack_output_external examples (#1109) +- [RESOLVE #946] Fixing bug preventing StackGroup dependencies (#1116) +- [Resolve #1138] Bugfix for j2_environments (#1137) +- [Resolve #1135] Fix path to templates (#1141) +- [Resolve #1143] fix "create" cmd with existing stack (#1144) +- [Resolves #1148] Correct path logic (#1149) + +### Nonfunctional + +- [Resolves #582] update imp to importlib (#1092) +- [Resolves #1090] Install troposphere as an extra package (#1104) +- [Resolves #1087] Add YAML document markers to template body (#1089) +- Stop hiding critical debug info in helpers (#988) (#997) +- [Resolves #1139] Provide useful info on invalid jinja file (#1142) + +## 2.6.3 (2021.08.13) + +### Fixed + +- [Resolves #1078] Fix delete CLI dependency tree + +## 2.6.2 (2021.08.02) + +### Fixed + +- [Resolves #1072] fix sceptre install for docker + +## 2.6.1 (2021.07.30) + +### Fixed + +- Fix dependencies to install sceptre-file-resolver +- Consolidate pip requirements files + +## 2.6.0 (2021.07.29) + +### Added + +- Doc: added docs to release workflow in .circleci/README.md +- Introduce an .editorconfig to sync common editor configuration across developer systems +- Update click version +- Update docker container to use Python 3.7 +- Make the sceptre-resolver-cmd resolver a core resolver +- Make the sceptre-file-resolver a core resolver + +### Removed + +- Doc: Removed V1 docs + +### Fixed + +- [Resolves #1013] Fix virtual-hosted-style uri +- Remove unnecessary padding with sceptre output command +- Do not open a web browser during test +- Optimize Sceptre Start Time by only Processing Dependent Configuration Files +- [Resolves #1042] Fix OS path resolution on windows + +## 2.5.0 (2021.05.01) + +### Added + +- Added support for python 3.8 & 3.9 +- Support PEP-518 builds with pyproject.toml file +- Support before_create_change_set and after_create_change_set hooks +- Allow generate command to run hooks + +### Removed + +- Removed support for python 2.7 & 3.5 + +### Fixed + +- Fix "sceptre list outpuuts" command +- Provide more info on config parsing errors + +### Nonfunctional + +- Removed Sonarqube +- Setup pre-commit linters and hooks +- General documentation updates + +## 2.4.0 (2020.10.03) + +### Added + +- Support for hooks on create_change_set + +### Fixed + +- Selection of correct stack group based on exact name +- Execution of empty change sets + +### Nonfunctional + +- Added jinja example to documentation +- Fixed documentation typos +- Tidied documentation by removing unnecessary comments +- Remove documented support for Python 2.7 + +## 2.3.0 (2020.02.03) + +### Added + +- `iam_role` capability for `stack-config` +- Support for complex data for `resolvers` and `hooks` +- Hooks support for `launch` command + +### Fixed + +- Replace JSON with YAML as default Cloudformation format in documentation +- Broken cross-page references in documentation +- Jinja autoescape vulnerability +- Connection manager imports +- `rel_paths` for `_call_sceptre_handler` in `template.py` +- Linting as a side-effect of upgrading flake package + +### Nonfunctional + +- Add documentation clean target +- Clean up code blocks in documentation +- Clarify pip vs. Docker installation in documentation +- Fix documentation version links +- Add autocomplete doc for ZSH shell +- Update integration test instructions +- Allow to keep specific versions of documentation +- Keep version `1.3.4` and `1.4.2` active on github pages +- Fix formatting error in `terminology.rst` +- Upgrade Dockerfile to Alpine 3.10 +- Improve error message for `stack_output` dependencies +- Improve `sceptre generate` formatting +- Unpinned some requirements to avoid conflict with other packages + ## 2.2.1 (2019.08.19) ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a38046ddf..5561e5ab4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,14 +8,15 @@ the project. This project adheres to the Contributor Covenant [code of conduct](http://contributor-covenant.org/version/1/4/). By participating, you are expected to uphold this code. Please report unacceptable -behaviour to sceptre@cloudreach.com. +behaviour to the [Sceptre github discussion](https://github.com/Sceptre/sceptre/discussions). # How to Contribute ## Report Bugs Before submitting a bug, please check our -[issues page](https://github.com/cloudreach/sceptre/issues) to see if it's +[issues page](https://github.com/Sceptre/sceptre/issues) and +[discussion board](https://github.com/Sceptre/sceptre/discussions) to see if it's already been reported. When reporting a bug, fill out the required template, and please include as much @@ -47,8 +48,7 @@ A good pull request: - Is clear. - Works across all supported version of Python. - Complies with the existing codebase style - ([flake8](http://flake8.pycqa.org/en/latest/), - [pylint](https://www.pylint.org/)). + ([pre-commit](https://pre-commit.com/)) - Includes [docstrings](https://www.python.org/dev/peps/pep-0257/) and comments for unintuitive sections of code. - Includes documentation for new features. @@ -77,11 +77,10 @@ $ git clone git@github.org:/sceptre.git 3. Install Sceptre for development (we recommend you use a [virtual environment](http://docs.python-guide.org/en/latest/dev/virtualenvs/)) +A convenient script is also provided: ```bash - $ cd sceptre/ - $ pip install -r requirements/prod.txt - $ pip install -r requirements/dev.txt - $ pip install -e . +$ cd sceptre +$ source venv.sh ``` 4. Create a branch for local development: @@ -90,36 +89,61 @@ $ git clone git@github.org:/sceptre.git $ git checkout -b ``` -5. When you're done making changes, check that your changes pass linting, unit - tests and have sufficient coverage and integration tests pass: +5. When you're done making changes, check that your changes pass + [linting](#Linting), [unit tests](#Unit-Tests) and have + sufficient coverage and [integration tests](#Integration-Tests) + pass. - Check linting: +6. Make sure the changes comply with the pull request guidelines in the section + on `Contributing Code`. -```bash -$ make lint -``` +7. Commit and push your changes. + + Commit messages should follow + [these guidelines](https://github.com/erlang/otp/wiki/Writing-good-commit-messages) + + Use the following commit message format + `[Resolves #issue_number] Short description of change`. + + e.g. `[Resolves #123] Fix description of resolver syntax in documentation` + +8. Submit a pull request through the GitHub website. + +## Linting + +As a pre-deployment step we syntatically validate files with +[pre-commit](https://pre-commit.com). + +Please [install pre-commit](https://pre-commit.com/#install) then run +`pre-commit install` to setup the git hooks. Once configured the pre-commit +linters will automatically run on every git commit. Alternatively you +can manually execute the validations by running `pre-commit run --all-files`. + +## Unit Tests Run unit tests or coverage in your current environment - (handy for quickly running unit tests): ```bash -$ make test $ make coverage +$ make test +$ make coverage ``` -Note: Sceptre aims to be compatible with Python 2 & 3, please run unit test -against both versions. You will need the corresponding versions of Python -installed on your system. +Note: Sceptre aims to be compatible with Python 3, please run unit test +against all supported versions. You will need the corresponding versions +of Python installed on your system. -Run unit tests and coverage using tox for Python 2.7, 3.6 and 3.7: +Run unit tests and coverage on all supported python versions: ```bash -$ tox -e py27 $ tox -e py36 -e py37 +$ make test-all ``` -If you use pyenv to manage Python versions, try `pip install tox-pyenv` to make -tox and pyenv play nicely. +Tox is used to manage python vresions for running unit tests. If you use pyenv +to manage Python versions, try `pip install tox-pyenv` to make tox and pyenv play +nicely. -Run integration tests: +## Integration Tests If you haven't setup your local environment or personal CircleCI account to run integration tests then follow these steps: @@ -141,6 +165,8 @@ your tests are passing): - Add your `Access Key ID` and `Secret Access Key` that is associated with an IAM User from your AWS account. The IAM User will require "Full" permissions for `CloudFormation` and `S3` and Write permissions for `STS` (AssumeRole). + For an example please take a look at the Sceptre + [CI service user policy](https://github.com/Sceptre/sceptre-aws/blob/master/config/prod/sceptre-integration-test-service-access.yaml#L5-L35) Once you have set up CircleCi any time you commit to a branch in your fork all tests will be run, including integration tests. @@ -148,43 +174,44 @@ tests will be run, including integration tests. You can also (optionally) run the integration tests locally, which is quicker during development. -To run integration tests locally: +### To run integration tests locally + * `pip install awscli` * Setup [AWS CLI Environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) to work with an AWS account that you have access to. You can use the same user that you use for CircleCi. * `pip install behave` -run: +_Note_: All integration tests are setup to run in `eu-west-*` region. If you prefer +to run in a different region you must update the region in each test before running it + +### run all tests ```bash $ behave integration-tests ``` -or to run a specific tests: +### run a specific feature ```bash -$ behave integration-tests -n "scenario-name" +$ behave integration-tests --include ``` -_Note_: All integration tests are setup to run in `eu-west-*` region. If you prefer -to run in a different region you must update the region in each test before running it. - - -6. Make sure the changes comply with the pull request guidelines in the section - on `Contributing Code`. - -7. Commit and push your changes. +### run a specific scenario - Commit messages should follow - [these guidelines](https://github.com/erlang/otp/wiki/Writing-good-commit-messages) - - Use the following commit message format - `[Resolves #issue_number] Short description of change`. +```bash +$ behave integration-tests -n "" +``` - e.g. `[Resolves #123] Fix description of resolver syntax in documentation` +# Sponsors -8. Submit a pull request through the GitHub website. +* [Sage Bionetworks](https://sagebionetworks.org/) donated the AWS account for running Sceptre integration + tests. Please contact it@sagebase.org for support. +* [GoDaddy](https://www.godaddy.com/) donated [the domain](https://docs.sceptre-project.org) for hosting + the Sceptre project. Please contact oss@godaddy.com for support. +* [Cloudreach](https://www.cloudreach.com/) started the Sceptre project and continuted to maintain it + until the ver 2.4 release. It has since been extricated from Cloudreach and has been maintained + by members of the Sceptre open source community. # Credits diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md deleted file mode 100644 index 6521fe8b1..000000000 --- a/CONTRIBUTORS.md +++ /dev/null @@ -1,47 +0,0 @@ -# Contributors - -# Project Lead - -- Niall Grant [@ngfgrant](https://github.com/ngfgrant) - -# Contributors - -- Nabeel Amjad [@nabeelamjad](https://github.com/nabeelamjad) - -- Henry Bell [@henrybell](https://github.com/henrybell) - -- Yvonne Beumer [@pinkeee](https://github.com/pinkeee) - -- Amy Bruce-Gardner [@amybg](https://github.com/amybg) - -- Chris Connelly [@connec](https://github.com/connec) - -- Max Eskenazi (max@eskenazi.net) - -- Adrian Fernandez [@afdezl](https://github.com/afdezl) - -- Douglas Fraser [@douglasfraser](https://github.com/douglasfraser) - -- Marius Galbinasu [@galbinasu](https://github.com/galbinasu) - -- Luke Plausin [@lukeplausin](https://github.com/lukeplausin) - -- Jan Rotter [@janrotter](https://github.com/janrotter) - -- James Routley [@jamesroutley](https://github.com/jamesroutley) - -- Sean Rankine [@theseanything](https://github.com/theseanything) - -- Stig Brautaset [@stig](https://github.com/stig) - -- Matthew Taylor [@matalo33](https://github.com/matalo33) - -- Michael Sverdlik [@m1keil](https://github.com/m1keil) - -- Olivier Van Goethem [@oliviervg1](https://github.com/oliviervg1) - -- Brendan Devenney [@devenney](https://github.com/devenney) - -- David Taddei [@davidtaddei](https://github.com/davidtaddei) - -- Dennis Conrad [@dennisconrad](https://github.com/dennisconrad) diff --git a/Dockerfile b/Dockerfile index 4e187f71b..81397b3e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.6.8-alpine3.10 +FROM python:3.7-alpine RUN apk add --no-cache bash WORKDIR /app COPY setup.cfg setup.py README.md CHANGELOG.md ./ COPY sceptre/ ./sceptre -RUN python setup.py install +RUN pip install . WORKDIR /project ENTRYPOINT ["sceptre"] diff --git a/Makefile b/Makefile index 121b7d39c..c78687c92 100644 --- a/Makefile +++ b/Makefile @@ -16,12 +16,10 @@ help: @echo "clean-build - remove build artifacts" @echo "clean-pyc - remove Python file artifacts" @echo "clean-test - remove test and coverage artifacts" - @echo "lint - check style with flake8" + @echo "lint - syntatic file validation with pre-commit" @echo "test - run tests quickly with the default Python" @echo "test-all - run tests on every Python version with tox" @echo "test-integration - run integration tests" - @echo "coverage - check code coverage quickly with the default Python" - @echo "coverage-ci - check code coverage and generate cobertura report" @echo "dist - package" @echo "install - install the package to the active Python's site-packages" @echo "install-dev - install the test requirements to the active Python's site-packages" @@ -53,36 +51,17 @@ clean-test: rm -f test-results.xml lint: - flake8 . + pre-commit run --all-files --show-diff-on-failure test: - pytest --junitxml=test-results/junit.xml + pytest test-all: - tox + tox --parallel=auto test-integration: install behave integration-tests/ -coverage-all: - coverage erase - coverage run --source sceptre -m pytest - coverage xml - -coverage: coverage-all - coverage report --show-missing --fail-under 92 - -sonar: - @sonar-scanner \ - -Dsonar.projectKey=Sceptre_${CIRCLE_PROJECT_REPONAME} \ - -Dsonar.organization=sceptre \ - -Dsonar.projectName=${CIRCLE_PROJECT_REPONAME} \ - -Dsonar.pullrequest.provider=GitHub\ - -Dsonar.branch.name=${CIRCLE_BRANCH}\ - -Dsonar.sources=. \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.login=${SONAR_LOGIN} - docs: rm -f docs/sceptre.rst rm -f docs/modules.rst diff --git a/README.md b/README.md index 5295c3af6..4825ad66d 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,21 @@ # Sceptre -[![Bugs](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=bugs)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=coverage)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=alert_status)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=security_rating)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -[![Technical Debt](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=sqale_index)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=Sceptre_sceptre&metric=vulnerabilities)](https://sonarcloud.io/dashboard?id=Sceptre_sceptre) -![image](https://circleci.com/gh/Sceptre/sceptre.png?style=shield) -![image](https://badge.fury.io/py/sceptre.svg) - -# About +[![CircleCI](https://img.shields.io/circleci/build/github/Sceptre/sceptre?logo=circleci)](https://app.circleci.com/pipelines/github/Sceptre) +[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/sceptreorg/sceptre?logo=docker&sort=semver)](https://hub.docker.com/r/sceptreorg/sceptre) +[![PyPI](https://img.shields.io/pypi/v/sceptre?logo=pypi)](https://pypi.org/project/sceptre/) +[![PyPI - Status](https://img.shields.io/pypi/status/sceptre?logo=pypi)](https://pypi.org/project/sceptre/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sceptre?logo=pypi)](https://pypi.org/project/sceptre/) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/sceptre?logo=pypi)](https://pypi.org/project/sceptre/) +[![License](https://img.shields.io/pypi/l/sceptre?logo=apache)](https://github.com/Sceptre/sceptre/blob/main/LICENSE) + +## About Sceptre is a tool to drive [AWS CloudFormation](https://aws.amazon.com/cloudformation). It automates the mundane, repetitive and error-prone tasks, enabling you to concentrate on building better infrastructure. -# Features +## Features - Code reuse by separating a Stack's template and its configuration - Support for templates written in JSON, YAML, Jinja2 or Python DSLs such as @@ -36,7 +33,7 @@ building better infrastructure. - Support for inserting dynamic values in templates via customisable Resolvers - Support for running arbitrary code as Hooks before/after Stack builds -# Benefits +## Benefits - Utilises cloud-native Infrastructure as Code engines (CloudFormation) - You do not need to manage state @@ -46,35 +43,35 @@ building better infrastructure. - Simple CLI and API - Unopinionated - Sceptre does not force a specific project structure -# Install +## Install -## Using pip +### Using pip `$ pip install sceptre` More information on installing sceptre can be found in our -[Installation Guide](https://sceptre.cloudreach.com/latest/docs/install.html) +[Installation Guide](https://docs.sceptre-project.org/latest/docs/install.html) -## Using Docker Image +### Using Docker Image -View our [Docker repository](https://hub.docker.com/r/cloudreach/sceptre). +View our [Docker repository](https://hub.docker.com/repositories/sceptreorg). Images available from version 2.0.0 onward. To use our Docker image follow these instructions: -1. Pull the image `docker pull cloudreach/sceptre:[SCEPTRE_VERSION_NUMBER]` e.g. - `docker pull cloudreach/sceptre:2.1.4`. Leave out the version number if you - wish to run `latest` or run `docker pull cloudreach/sceptre:latest`. +1. Pull the image `docker pull sceptreorg/sceptre:[SCEPTRE_VERSION_NUMBER]` e.g. + `docker pull sceptreorg/sceptre:2.5.0`. Leave out the version number if you + wish to run `latest` or run `docker pull sceptreorg/sceptre:latest`. 2. Run the image. You will need to mount the working directory where your project resides to a directory called `project`. You will also need to mount a volume with your AWS config to your docker container. E.g. -`docker run -v $(pwd):/project -v /Users/me/.aws/:/root/.aws/:ro cloudreach/sceptre:latest --help` +`docker run -v $(pwd):/project -v /Users/me/.aws/:/root/.aws/:ro sceptreorg/sceptre:latest --help` If you want to use a custom ENTRYPOINT simply amend the Docker command: -`docker run -ti --entrypoint='' cloudreach:latest sh` +`docker run -ti --entrypoint='' sceptreorg/sceptre:latest sh` The above command will enter you into the shell of the Docker container where you can execute sceptre commands - useful for development. @@ -83,25 +80,13 @@ If you have any other environment variables in your non-docker shell you will need to pass these in on the Docker CLI using the `-e` flag. See Docker documentation on how to achieve this. -# Migrate v1 to v2 - -We have tried to make the migration to Sceptre v2 as simple as possible. For -information about how to migration your v1 project please see our -[Migration Guide](https://github.com/sceptre/project/wiki/Migration-Guide:-V1-to-V2) - -# V1 End of Life Notice - -Support for Version 1 will -[end on June 1 2019](https://github.com/sceptre/sceptre/issues/593). For new -projects we recommend using Version 2. - -# Example +## Example Sceptre organises Stacks into "Stack Groups". Each Stack is represented by a YAML configuration file stored in a directory which represents the Stack Group. Here, we have two Stacks, `vpc` and `subnets`, in a Stack Group named `dev`: -``` +```sh $ tree . ├── config @@ -117,7 +102,7 @@ $ tree We can create a Stack with the `create` command. This `vpc` Stack contains a VPC. -``` +```sh $ sceptre create dev/vpc.yaml dev/vpc - Creating stack dev/vpc @@ -131,7 +116,7 @@ this, we need to pass the VPC ID, which is exposed as a Stack output of the `vpc` Stack, to a parameter of the `subnets` Stack. Sceptre automatically resolves this dependency for us. -``` +```sh $ sceptre create dev/subnets.yaml dev/subnets - Creating stack dev/subnets Subnet AWS::EC2::Subnet CREATE_IN_PROGRESS @@ -142,7 +127,7 @@ dev/subnets sceptre-demo-dev-subnets AWS::CloudFormation::Stack CREATE_COMPLETE Sceptre implements meta-operations, which allow us to find out information about our Stacks: -``` +```sh $ sceptre list resources dev/subnets.yaml - LogicalResourceId: Subnet @@ -156,7 +141,7 @@ Sceptre provides Stack Group level commands. This one deletes the whole `dev` Stack Group. The subnet exists within the vpc, so it must be deleted first. Sceptre handles this automatically: -``` +```sh $ sceptre delete dev Deleting stack @@ -174,7 +159,7 @@ dev/vpc - Stack deleted Sceptre can also handle cross Stack Group dependencies, take the following example project: -``` +```sh $ tree . ├── config @@ -213,22 +198,25 @@ Sceptre can be used from the CLI, or imported as a Python package. ## CLI -``` +```text Usage: sceptre [OPTIONS] COMMAND [ARGS]... Sceptre is a tool to manage your cloud native infrastructure deployments. Options: - --version Show the version and exit. - --debug Turn on debug logging. - --dir TEXT Specify sceptre directory. - --output [yaml|json] The formatting style for command output. - --no-colour Turn off output colouring. - --var TEXT A variable to template into config files. - --var-file FILENAME A YAML file of variables to template into config - files. - --ignore-dependencies Ignore dependencies when executing command. - --help Show this message and exit. + --version Show the version and exit. + --debug Turn on debug logging. + --dir TEXT Specify sceptre directory. + --output [text|yaml|json] The formatting style for command output. + --no-colour Turn off output colouring. + --var TEXT A variable to replace the value of an item in + config file. + --var-file FILENAME A YAML file of variables to replace the values + of items in config files. + --ignore-dependencies Ignore dependencies when executing command. + --merge-vars Merge variables from successive --vars and var + files. + --help Show this message and exit. Commands: create Creates a stack or a change set. @@ -269,13 +257,31 @@ plan.launch() ``` Full API reference documentation can be found in the -[Documentation](https://sceptre.cloudreach.com/) +[Documentation](https://docs.sceptre-project.org/) ## Tutorial and Documentation -- [Get Started](https://sceptre.cloudreach.com/latest/docs/get_started.html) -- [Documentation](https://sceptre.cloudreach.com/) +- [Get Started](https://docs.sceptre-project.org/latest/docs/get_started.html) +- [Documentation](https://docs.sceptre-project.org/) + +## Communication + +Sceptre community discussions happen in the #sceptre chanel in the +[og-aws Slack](https://github.com/open-guides/og-aws). To join click +on to create an account and join the +#sceptre channel. + +Follow the [SceptreOrg Twitter account](https://twitter.com/SceptreOrg) to get announcements on the latest releases. ## Contributing See our [Contributing Guide](CONTRIBUTING.md) + + +## Sponsors + +[![Sage Bionetworks](sponsors/sage_bionetworks_logo.png "Sage Bionetworks")](https://sagebionetworks.org) + +[![GoDaddy](sponsors/godaddy_logo.png "GoDaddy")](https://www.godaddy.com) + +[![Cloudreach](sponsors/cloudreach_logo.png "Cloudreach")](https://www.cloudreach.com) diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..c5fa5f982 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,13 @@ +# Sceptre release process + +1. Bump version in `./sceptre/__init__.py` & `./setup.cfg` +2. Update Changelog with details from Github commit list since last release +3. Create a PR for above changes +4. Once PR is merged, `git pull` the changes to sync the *master* branch +5. `git tag -as vX.Y.Z` +6. `git push origin vX.Y.Z` (CI/CD publishes to PyPi) +7. Get list of contributors with + `git log --no-merges --format='%<(20)%an' v1.0.0..HEAD | sort | uniq`, where + the tag is the last deployed version. +8. Announce release to the #sceptre channel on og-aws Slack with a link to + the latest changelog and list of contributors diff --git a/docs/_source/_templates/versions.html b/docs/_source/_templates/versions.html index 738f312a4..c1e61a1cb 100644 --- a/docs/_source/_templates/versions.html +++ b/docs/_source/_templates/versions.html @@ -13,7 +13,7 @@
{% set links = { 'Cloudreach':'https://www.cloudreach.com', - 'Sceptre Home': 'https://www.sceptre.cloudreach.com' + 'Sceptre Home': 'https://docs.sceptre-project.org' } %}
Other Links
{% for title, url in links.items() %} @@ -39,4 +39,4 @@ }); -{% endif %} \ No newline at end of file +{% endif %} diff --git a/docs/_source/about.rst b/docs/_source/about.rst index 858c8849c..468503ccb 100644 --- a/docs/_source/about.rst +++ b/docs/_source/about.rst @@ -39,8 +39,15 @@ Sceptre’s source code can be found on `Github`_. Bugs and feature requests should be raised via our `Issues`_ page. -.. _Github: https://github.com/cloudreach/sceptre/ -.. _Issues: https://github.com/cloudreach/sceptre/issues +Communication +------------- + +The Sceptre community uses a Slack channel #sceptre on the og-aws Slack for +discussion. To join use this link http://slackhatesthe.cloud/ to create an +account and join the #sceptre channel. + +.. _Github: https://github.com/Sceptre/sceptre/ +.. _Issues: https://github.com/Sceptre/sceptre/issues .. _CloudFormation: https://aws.amazon.com/cloudformation/ .. _AWS CLI: https://aws.amazon.com/cli/ .. _Boto3: https://aws.amazon.com/sdk-for-python/ diff --git a/docs/_source/apidoc/sceptre.cli.rst b/docs/_source/apidoc/sceptre.cli.rst index e322e2ede..7e5224bf1 100644 --- a/docs/_source/apidoc/sceptre.cli.rst +++ b/docs/_source/apidoc/sceptre.cli.rst @@ -25,6 +25,14 @@ sceptre.cli.delete module :undoc-members: :show-inheritance: +sceptre.cli.diff module +_______________________ + +.. automodule:: sceptre.cli.diff + :members: + :undoc-members: + :show-inheritance: + sceptre.cli.describe module --------------------------- @@ -104,5 +112,3 @@ sceptre.cli.update module :members: :undoc-members: :show-inheritance: - - diff --git a/docs/_source/apidoc/sceptre.config.rst b/docs/_source/apidoc/sceptre.config.rst index b1d2a44f1..ed86e9649 100644 --- a/docs/_source/apidoc/sceptre.config.rst +++ b/docs/_source/apidoc/sceptre.config.rst @@ -32,5 +32,3 @@ sceptre.config.strategies module :members: :undoc-members: :show-inheritance: - - diff --git a/docs/_source/apidoc/sceptre.diffing.rst b/docs/_source/apidoc/sceptre.diffing.rst new file mode 100644 index 000000000..cac01854b --- /dev/null +++ b/docs/_source/apidoc/sceptre.diffing.rst @@ -0,0 +1,26 @@ +sceptre.diffing package +======================= + +.. automodule:: sceptre.diffing + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +sceptre.diffing.stack_differ module +----------------------------------- + +.. automodule:: sceptre.diffing.stack_differ + :members: + :undoc-members: + :show-inheritance: + +sceptre.diffing.diff_writer module +---------------------------------- + +.. automodule:: sceptre.diffing.diff_writer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/_source/apidoc/sceptre.hooks.rst b/docs/_source/apidoc/sceptre.hooks.rst index 70bf0a39c..442566e15 100644 --- a/docs/_source/apidoc/sceptre.hooks.rst +++ b/docs/_source/apidoc/sceptre.hooks.rst @@ -24,5 +24,3 @@ sceptre.hooks.cmd module :members: :undoc-members: :show-inheritance: - - diff --git a/docs/_source/apidoc/sceptre.plan.rst b/docs/_source/apidoc/sceptre.plan.rst index b23f7b0fb..69131373a 100644 --- a/docs/_source/apidoc/sceptre.plan.rst +++ b/docs/_source/apidoc/sceptre.plan.rst @@ -32,5 +32,3 @@ sceptre.plan.plan module :members: :undoc-members: :show-inheritance: - - diff --git a/docs/_source/apidoc/sceptre.resolvers.rst b/docs/_source/apidoc/sceptre.resolvers.rst index f824972ed..1c50323a6 100644 --- a/docs/_source/apidoc/sceptre.resolvers.rst +++ b/docs/_source/apidoc/sceptre.resolvers.rst @@ -32,5 +32,3 @@ sceptre.resolvers.stack\_output module :members: :undoc-members: :show-inheritance: - - diff --git a/docs/_source/apidoc/sceptre.rst b/docs/_source/apidoc/sceptre.rst index 6c47dc332..0da359623 100644 --- a/docs/_source/apidoc/sceptre.rst +++ b/docs/_source/apidoc/sceptre.rst @@ -13,6 +13,7 @@ Subpackages sceptre.cli sceptre.config + sceptre.diffing sceptre.hooks sceptre.plan sceptre.resolvers @@ -83,5 +84,3 @@ sceptre.template module :members: :undoc-members: :show-inheritance: - - diff --git a/docs/_source/conf.py b/docs/_source/conf.py index f242ae6b2..4fe77fd2d 100644 --- a/docs/_source/conf.py +++ b/docs/_source/conf.py @@ -19,7 +19,7 @@ import os import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..' + os.path.sep + '..')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".." + os.path.sep + "..")) import sceptre # noqa # The short X.Y version @@ -27,8 +27,8 @@ # The full version, including alpha/beta/rc tags release = version -project = 'Sceptre' -copyright = '2018, Cloudreach' +project = "Sceptre" +copyright = "2018, Cloudreach" author = sceptre.__author__ # -- General configuration --------------------------------------------------- @@ -41,33 +41,34 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx_click.ext', + "sphinx.ext.autodoc", + "sphinx_autodoc_typehints", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinx_click.ext", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set 'language' from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -82,7 +83,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -109,7 +110,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'Sceptredoc' +htmlhelp_basename = "Sceptredoc" # -- Options for LaTeX output ------------------------------------------------ @@ -132,14 +133,14 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'Sceptre.tex', 'Sceptre Documentation', 'Cloudreach', 'manual') + (master_doc, "Sceptre.tex", "Sceptre Documentation", "Cloudreach", "manual") ] # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, 'sceptre', 'Sceptre Documentation', [author], 1)] +man_pages = [(master_doc, "sceptre", "Sceptre Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -149,12 +150,12 @@ texinfo_documents = [ ( master_doc, - 'Sceptre', - 'Sceptre Documentation', + "Sceptre", + "Sceptre Documentation", author, - 'Sceptre', - 'One line description of project.', - 'Miscellaneous', + "Sceptre", + "One line description of project.", + "Miscellaneous", ) ] @@ -176,7 +177,7 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- @@ -184,11 +185,12 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'python': ('https://docs.python.org/', None), - 'boto3': ( - 'https://boto3.readthedocs.io/en/latest/', - 'https://boto3.readthedocs.io/en/latest/objects.inv', + "python": ("https://docs.python.org/3/", None), + "boto3": ( + "https://boto3.amazonaws.com/v1/documentation/api/latest/", + "https://boto3.amazonaws.com/v1/documentation/api/latest/objects.inv", ), + "deepdiff": ("https://zepworks.com/deepdiff/current/", None), } # other configuration @@ -196,6 +198,17 @@ nitpicky = True nitpick_ignore = [ - ('py:class', 'json.encoder.JSONEncoder'), - ('py:class', 'sceptre.config.reader.Attributes') + ("py:class", "json.encoder.JSONEncoder"), + ("py:class", "sceptre.config.reader.Attributes"), + ("py:class", "sceptre.diffing.stack_differ.DiffType"), + ("py:obj", "sceptre.diffing.stack_differ.DiffType"), + ("py:class", "DiffType"), + ("py:class", "TextIO"), + ("py:class", "_io.StringIO"), + ("py:class", "yaml.loader.SafeLoader"), + ("py:class", "yaml.dumper.Dumper"), + ("py:class", "cfn_tools.odict.ODict"), + ("py:class", "T_Container"), ] + +set_type_checking_flag = True diff --git a/docs/_source/docs/cli.rst b/docs/_source/docs/cli.rst index 55c5cd9b2..8075fe382 100644 --- a/docs/_source/docs/cli.rst +++ b/docs/_source/docs/cli.rst @@ -74,6 +74,35 @@ we could overwrite ``nested: world`` to ``nested: hi`` using: a dependency. Using a --var-file with all variables set can help meet this requirement. +It is also possible to have keys merged according to a deep merge +algorithm from successive var files, by specifying ``--merge-vars``. So, if we +had a second variable file "vars2.yaml": + +.. code-block:: yaml + + # other_vars.yaml + --- + top: + middle3: + nested: more world + + +We could merge all of this together using: + +``sceptre --merge-vars --var-file vars.yaml --var-file other_vars.yaml launch stack`` + +The ``top`` dictionary would then be expected to contain: + +.. code-block:: python + + { + "top": { + "middle": {"nested": "hello"}, + "middle2": {"nested": "world"}, + "middle3": {"nested": "more world"} + } + } + Command reference ----------------- diff --git a/docs/_source/docs/faq.rst b/docs/_source/docs/faq.rst index 38cec1f9a..6563b0cce 100644 --- a/docs/_source/docs/faq.rst +++ b/docs/_source/docs/faq.rst @@ -82,3 +82,85 @@ built in the same ``launch`` command, the environment variable resolver must be used. .. _AWS documentation: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html + +.. _using_connection_manager: + +How do I call AWS services or use AWS-based tools in my custom hook/resolver/template handler? +---------------------------------------------------------------------------------------------- +In order to call AWS services in your custom hook/resolver/template handler properly, you should use +the IAM configurations of the stack where the resolver is being used (unless you need to use a +different configuration for a specific reason). This means your hook/resolver/handler should honor the +``profile``, ``region``, and ``sceptre_role`` configurations as set for your project and/or Stack Config. +Simply invoking ``boto3.client('s3')`` is _not_ going to regard those and could end up using the +wrong credentials or not even working. + +There is a simple interface available for doing this properly, the +:py:class:`sceptre.connection_manager.ConnectionManager`. The ConnectionManager is an interface that +will be pre-configured with each stack's profile, region, and sceptre_role and will be ready for you to use. +If you are using an ``sceptre_role``, it will automatically assume that role via STS for making calls to +AWS so you can just use it the way you want. It is accessible on hooks and resolvers via +``self.stack.connection_manager`` and on template_handlers via ``self.connection_manager``. + +There are three public methods on the ConnectionManager: + +- :py:meth:`sceptre.connection_manager.ConnectionManager.call` can be used to directly call a boto3 + API method on any api service and return the result from boto3. This is perfect when you just need + to invoke a boto3 client method without any additional capabilities. +- :py:meth:`sceptre.connection_manager.ConnectionManager.get_session` can be used to get a boto3 Session + object. This is very useful if you need to work with Boto3 Resource objects (like an s3 Bucket) or + if you need to create and pass the bot3 session, client, or resource to a third-party framework. +- :py:meth:`sceptre.connection_manager.ConnectionManager.create_session_environment_variables` creates + a dictionary of environment variables used by AWS sdks with all the relevant connection information. + This is extremely useful if you are needing to invoke other SDKs using ``subprocess`` and still need + the Stack's connection information honored. + + +Using the connection manager, you can use `boto3 `_ +to perform any AWS actions you need: + +.. code-block:: python + + # For example, in your custom resolver: + def resolve(self): + # You can invoke a lower-level service method like... + obj = self.stack.connection_manager.call('s3', 'get_object', {'Bucket': 'my-bucket', 'Key': 'my-key'}) + # Or you can create higher-level resource objects like... + bucket = self.stack.connection_manager.get_session().resource('s3').Bucket('my-bucket') + # Or if you need to invoke a third-party tool via a subprocess, you can create the necessary environment + # variables like this: + environment_variables = self.stack.connection_manager.create_session_environment_variables( + include_system_envs=True + ) + list_output = subprocess.run( + 'aws s3 list-bucket', + shell=True, + env=environment_variables, + capture_output=True + ).stdout + + +My CI/CD process uses ``sceptre launch``. How do I delete stacks that aren't needed anymore? +--------------------------------------------------------------------------------------------- + +Running the ``launch`` command is a very useful "1-stop-shop" to apply changes from Stack Configs, +creating stacks that don't exist and updating stacks that do exist. This makes it a very useful +command to configure your CI/CD system to invoke. However, sometimes you need to delete a stack that +isn't needed anymore and you want this automatically applied by the same process. + +This "clean up" is complicated by the fact that Sceptre doesn't know anything that isn't in its +Stack and StackGroup Configs; If you delete a Stack Config, Sceptre won't know to clean it up. + +Therefore, the way to accomplish this "clean up" operation is to perform the change in 3 steps: + +1. First, add ``obsolete: True`` to the Stack Config(s) you want to clean up. + For more information on ``obsolete``, see the :ref:`Stack Config entry on it`. +2. Update your CI/CD process to run ``sceptre launch --prune`` instead of ``sceptre launch``. This + will cause all stacks marked as obsolete to be deleted going forward. +3. Once your CI/CD process has cleaned up all the obsolete stacks, delete the local Stack Config files + you marked as obsolete in step 1, since the stacks they create have all been deleted. + +.. note:: + + Using ``obsolete: True`` will not work if any other stacks depend on that stack that are + not themselves obsolete. Attempting to prune any obsolete stacks that are depended on by + non-obsolete stacks will result in Sceptre immediately failing the launch. diff --git a/docs/_source/docs/get_started.rst b/docs/_source/docs/get_started.rst index 85f86b732..7ea8d85db 100644 --- a/docs/_source/docs/get_started.rst +++ b/docs/_source/docs/get_started.rst @@ -7,6 +7,16 @@ Install This tutorial assumes that you have installed Sceptre. Instructions on how to do this can be found in the section on :doc:`installing Sceptre `. +AWS CLI Config +-------------- + +Sceptre uses the same configuration files as the official AWS CLI and can be +configured using the `aws configure` command. This command configures 2 files +`~/.aws/config` and `~/.aws/credentials` to setup the API Keys to access your +AWS Account. + +Reference: `AWS_CLI_Configure`_ + Directory Structure ------------------- @@ -96,13 +106,15 @@ Add the following configuration to ``config/dev/vpc.yaml``: .. code-block:: yaml - template_path: vpc.yaml + template: + path: vpc.yaml + type: file parameters: CidrBlock: 10.0.0.0/16 -``template_path`` specifies the relative path to the CloudFormation, Python or +``template`` specifies the relative path to the CloudFormation, Python or Jinja2 template to use to launch the Stack. Sceptre will use the ``templates`` -directory as the root templates directory to base your ``template_path`` from. +directory as the root templates directory to base your ``path`` from. ``parameters`` lists the parameters which are supplied to the template ``vpc.yaml``. @@ -124,7 +136,7 @@ You should now have a Sceptre project that looks a bit like: .. Note: You do not need to make sure the Template and Stack config names - match, since you define the ``template_path`` in your Stack config, but it + match, since you define the ``template`` in your Stack config, but it can be useful to keep track of what is going on. You will also notice that we have two ``config.yaml`` files, one in ``config/`` @@ -185,3 +197,4 @@ reference to the CLI :doc:`in our CLI guide ` .. _CloudFormation: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/Welcome.html +.. _AWS_CLI_Configure: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html diff --git a/docs/_source/docs/hooks.rst b/docs/_source/docs/hooks.rst index 81ab4ce74..9adb2fbd4 100644 --- a/docs/_source/docs/hooks.rst +++ b/docs/_source/docs/hooks.rst @@ -12,12 +12,20 @@ If required, users can create their own ``hooks``, as described in the section Hook points ----------- +``before_generate`` or ``after_generate`` - run hook before or after generating stack template. + ``before_create`` or ``after_create`` - run hook before or after Stack creation. ``before_update`` or ``after_update`` - run hook before or after Stack update. ``before_delete`` or ``after_delete`` - run hook before or after Stack deletion. +``before_launch`` or ``after_launch`` - run hook before or after Stack launch. + +``before_validate`` or ``after_validate`` - run hook before or after Stack validation. + +``before_create_change_set`` or ``after_create_change_set`` - run hook before or after create change set. + Syntax: Hooks are specified in a Stack’s config file, using the following syntax: @@ -83,7 +91,9 @@ specified: .. code-block:: yaml - template_path: templates/example.py + template: + path: templates/example.py + type: file parameters: ExampleParameter: example_value hooks: @@ -146,9 +156,6 @@ custom_hook.py argument defined in the Sceptre config file (see below) stack: sceptre.stack.Stack The associated stack of the hook. - connection_manager: sceptre.connection_manager.ConnectionManager - Boto3 Connection Manager - can be used to call boto3 api. - """ def __init__(self, *args, **kwargs): super(CustomHook, self).__init__(*args, **kwargs) @@ -191,12 +198,44 @@ This hook can be used in a Stack config file with the following syntax: .. code-block:: yaml - template_path: <...> + template: + path: <...> + type: <...> hooks: before_create: - !custom_hook_command_name # The argument is accessible via self.argument +hook arguments +^^^^^^^^^^^^^^ +Hook arguments can be a simple string or a complex data structure. You can even use resolvers in +hook arguments, so long as they're nested in a list or a dict. + +Assume a Sceptre `copy` hook that calls the `cp command`_: + +.. code-block:: yaml + + template: + path: <...> + type: <...> + hooks: + before_create: + - !copy "-r from_dir to_dir" + before_update: + - !copy {"options":"-r", "source": "from_dir", "destination": "to_dir"} + after_update: + - !copy + options: "-r" + source: "from_dir" + destination: !stack_output my/other/stack::CopyDestination + .. _Custom Hooks: #custom-hooks .. _subprocess documentation: https://docs.python.org/3/library/subprocess.html .. _documentation: http://docs.aws.amazon.com/autoscaling/latest/userguide/as-suspend-resume-processes.html -.. _this is great place to start: https://docs.python.org/3/distributing/ \ No newline at end of file +.. _this is great place to start: https://docs.python.org/3/distributing/ +.. _cp command: http://man7.org/linux/man-pages/man1/cp.1.html + +Calling AWS services in your custom hook +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For details on calling AWS services or invoking AWS-related third party tools in your hooks, see +:ref:`using_connection_manager` diff --git a/docs/_source/docs/permissions.rst b/docs/_source/docs/permissions.rst new file mode 100644 index 000000000..44c9faded --- /dev/null +++ b/docs/_source/docs/permissions.rst @@ -0,0 +1,184 @@ +Sceptre and IAM +=============== + +Inevitably, when working with CloudFormation, IAM permissions are relevant. **By default, Sceptre +uses the permissions of the user executing Sceptre commands when performing any of its actions.** +Also, by default, CloudFormation uses the permissions of the user executing its actions when +performing actions on individual resources within those stacks. + +This means that, by default, if the user executing Sceptre has admin-level permissions, there won't +be any issue executing any actions. But this is often not a viable option. Organizations usually have +few users with these permissions. Furthermore, there are legitimate reasons for not wanting to grant +users (or CI/CD systems like Jenkins) admin-level permissions. + +Of course, admin-level AWS permissions aren't essential; At the end of the day, Sceptre and +CloudFormation actually only need the permissions necessary to perform the required actions for a +given project on the resources that will be managed by them. The actual range of permissions required +tends to be a much narrower scope than admin-level. + +Permissions Configurations +-------------------------- + +There are three main configurations for Sceptre that can modify default permissions behavior to +provide more flexibility, control, and safety within an organization: **cloudformation_service_role**, +**sceptre_role**, and **profile**. These can be applied in a very targeted way, on a stack by stack +basis or can be applied broadly to a whole StackGroup. + +.. _cloudformation_service_role_permissions: + +cloudformation_service_role +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +This is the **CloudFormation service role** that will be attached to a given CloudFormation stack. +This IAM role needs to be able to be assumed by **CloudFormation**, and must provide all the +necessary permissions for all create/read/update/delete operations for all resources defined in that +stack. If CloudFormation can assume this role, the user executing Sceptre does not need those +permissions, only ``iam:PassRole`` permissions on that role to give it to CloudFormation. + +There are a few very important things to note about using a CloudFormation service role: + +* You cannot remove a role from a stack once you've added it. You'd have to delete and rebuild the + stack without it if you wanted to remove it. You *may*, however, replace one role with another on + the stack. +* Once the role has been added to a stack, it will *always* be used for all actions; It doesn't matter + who is executing them; those permission cannot be overridden without modifying the role itself or + replacing it with another role. +* You cannot delete a stack with a service role unless that role has permissions to delete those + resources. This means that an admin might need to add deletion permissions to that role before that + stack can be removed. + +Applying a service role to a stack is a very effective way to grant (and limit) the scope of permissions +that a stack can utilize, but the rigidity of using it might prove overly burdensome, depending on +the use case. + +For more information on using CloudFormation service roles, see the `AWS documentation `_. + +As a resolvable property, Sceptre allows you to use a resolver to populate the ``cloudformation_service_role`` for a +Stack or StackGroup Config. This means you could define that role within your project, output its +ARN, and then reference it using `!stack_output`. + +.. _sceptre_role_permissions: + +sceptre_role +^^^^^^^^^^^^ + +This is a **role that Sceptre will assume** when taking any actions on the Stack. It is not a service +role for CloudFormation. Instead, this is simply a role that the current user assumes to execute +any Sceptre actions on that stack. This has some benefits over a CloudFormation service role: + +* This is not permanently attached to the Stack after you've used it once. If you ever *don't* want + to assume this role, you could comment it out or remove it from the Stack Config and Sceptre simply + won't use it. This is useful if the user executing Sceptre already has the right permissions to + take those actions. In other words, it doesn't lock you in (unlike using ``cloudformation_service_role``). +* CloudFormation can continue to use it's default behavior of executing Stack actions with the + permissions of the current user, but it interprets the current user to hold the indicated ``sceptre_role``, + which could grant additional permissions. + +Using the ``sceptre_role`` configuration on a Stack or StackGroup Config allows the user to *temporarily* +"step into" a different set of permissions in order to execute Sceptre actions on Stack(s) in the +project without having to permanently hold those permissions. + +In order to use an ``sceptre_role`` on a Sceptre Stack Config, that role needs to have an +AssumeRolePolicyDocument that allows the current user to assume it and permissions to perform all +deployment actions on the stack and all its resources. + +As a resolvable property, Sceptre allows you to use a resolver to populate the ``sceptre_role`` for a +Stack or StackGroup Config. This means you could define that role within your project, output its +ARN, and then reference it using `!stack_output`. + +.. _profile_permissions: + +profile +^^^^^^^ + +This is different from ``cloudformation_service_role`` and ``sceptre_role``, as both of those cause CloudFormation or +Sceptre to *assume* a different role with different permissions than the permissions the current +user has. + +In contrast, the ``profile`` is simply an instruction for the underlying AWS SDK to reference that +profile in the user's local AWS configuration. It indicates which set of *credentials* to use when +operating on a given Stack or StackGroup. There is no call to AWS STS to assume a role temporarily. + +Utilizing the ``profile`` configuration is identical to setting the ``AWS_PROFILE`` environment +variable, which has the same effect. + +Tips for working with Sceptre, IAM, and a CI/CD system +------------------------------------------------------ + +* Rather than giving your CI/CD system blanket, admin-level permissions, you can define an IAM role + with Sceptre to use for deploying the rest of your infrastructure, outputing its ARN in the template. + Then, in the rest of your project's stacks, you can set the ``sceptre_role`` using ``!stack_output`` + to get that role's arn. This will mean your CI/CD system will temporarily "step into" that role + when using Sceptre to interact with those specific stacks. It will also establish a dependency on + your deployment role stack with every other stack in your project. + +* You can constrain the permissions of that deployment role by using: + + * An ``AssumeRolePolicyDocument`` that only allows the CI/CD system to assume it (instead of just + anyone in your organization). + * A ``PermissionsBoundary`` that guards sensitive/critical infrastructure and which must be on + any roles created/updated by the deployment role. See `AWS documentation on permission boundaries + `_ for more + information. + * The ``Path`` on IAM roles and managed policies to namespace your resources. Since that path is + a part of the ARN structure on roles and managed policies, you can constrain the IAM-related + permissions of the deployment role to only certain paths, preventing the deployment role from + elevating its own permissions or modifying unrelated roles and policies. + * Using ``aws:CalledVia`` and ``aws:CalledViaFirst`` conditions matching against + ``"cloudformation.amazonaws.com"`` to ensure that the deployment role can only execute changes + via CloudFormation and not on its own. Note: Some actions are taken by Sceptre directly and not + via cloudformation (see the section below on this). Those actions should *not* have a CalledVia + condition applied. + +* If you define your deployment role (and any other related resources) using Sceptre and then + reference it on all *other* stacks using ``sceptre_role: !stack_output ...``, this means that your + CI/CD system will not be able to deploy changes to the deployment role or its resources, but that + every deployment will depend on those. This is good! It means that, so long as those resources + remain unchanged, automated deployment can proceed without issue. It also means that the scope of + powers held by the deployment role needs to be reviewed by and **manually deployed by a user with + admin-level permissions.** But after that manual deployment, your CI/CD system should be empowered + to deploy all the other stacks in your project (so long as the deployment role has the full scope of + permissions needed to do those deployments). + +Basic permissions that Sceptre requires +--------------------------------------- + +There are certain permissions that Sceptre requires to perform even its most basic operations. These +include: + +**For Basic operations:** + +* cloudformation:CreateStack +* cloudformation:DeleteStack +* cloudformation:DescribeStackEvents +* cloudformation:DescribeStackResource +* cloudformation:DescribeStackResources +* cloudformation:DescribeStacks +* cloudformation:GetStackPolicy +* cloudformation:GetTemplate +* cloudformation:GetTemplateSummary +* cloudformation:ListStackResources +* cloudformation:ListStacks +* cloudformation:SetStackPolicy +* cloudformation:TagResource +* cloudformation:UntagResource +* cloudformation:UpdateStack +* cloudformation:UpdateTerminationProtection +* cloudformation:ValidateTemplate + +**If using change sets:** + +* cloudformation:CreateChangeSet +* cloudformation:DeleteChangeSet +* cloudformation:DescribeChangeSet +* cloudformation:ExecuteChangeSet +* cloudformation:ListChangeSets + +**If using a template bucket:** + +* s3:CreateBucket +* s3:PutObject + +**If using a cloudformation service role:** + +* iam:PassRole diff --git a/docs/_source/docs/resolvers.rst b/docs/_source/docs/resolvers.rst index ebca24215..97080123a 100644 --- a/docs/_source/docs/resolvers.rst +++ b/docs/_source/docs/resolvers.rst @@ -5,6 +5,9 @@ Sceptre implements resolvers, which can be used to resolve a value of a CloudFormation ``parameter`` or ``sceptre_user_data`` value at runtime. This is most commonly used to chain the outputs of one Stack to the inputs of another. +You can use resolvers with any resolvable property on a StackConfig, as well as in the arguments +of hooks and other resolvers. + If required, users can create their own resolvers, as described in the section on `Custom Resolvers`_. @@ -39,24 +42,155 @@ Example: parameters: database_password: !environment_variable DATABASE_PASSWORD +file +~~~~ + +A Sceptre resolver to get file contents. The returned value can be passed into a parameter as +a string, json, or yaml object. + +Refer to `sceptre-file-resolver `_ for documentation. + file_contents ~~~~~~~~~~~~~ -Reads in the contents of a file. +**deprecated**: Consider using the `file`_ resolver instead. -Syntax: +join +~~~~ + +This resolver allows you to join multiple strings together to form a single string. This is great +for combining the outputs of multiple resolvers. This resolver works just like CloudFormation's +``!Join`` intrinsic function. + +The argument for this resolver should be a list with two elements: (1) A string to join the elements +on and (2) a list of items to join. + +Example: .. code-block:: yaml - parameters|sceptre_user_data: - : !file_contents /path/to/file.txt + parameters: + BaseUrl: !join + - ":" + - - !stack_output my/app/stack.yaml::HostName + - !stack_output my/other/stack.yaml::Port + + +no_value +~~~~~~~~ + +This resolver "resolves to nothing", functioning just as if it was not set at all. This works just +like the "AWS::NoValue" special variable that you can reference on a CloudFormation template. It +can help simplify Stack and StackGroup config Jinja logic in cases where, if a condition is met, a +value is passed, otherwise no value is passed. + +For example, you could use this resolver like this: + +.. code-block:: yaml + + parameters: + my_parameter: {{ var.some_value_that_might_not_be_set | default('!no_value') }} + +In this example, if ``var.some_value_that_might_not_be_set`` is set, ``my_parameter`` will be set to +that value. But if ``var.some_value_that_might_not_be_set`` is not actually set, ``my_parameter`` +won't even be passed to CloudFormation at all. This might be desired if there is a default value on +the CloudFormation template for ``my_parameter`` and we'd want to fall back to that default. + +rcmd +~~~~ + +A resolver to execute any shell command. + +Refer to `sceptre-resolver-cmd `_ for documentation. + +select +~~~~~~ + +This resolver allows you to select a specific index of a list of items. This is great for combining +with the ``!split`` resolver to obtain part of a string. This function works almost the same as +CloudFormation's ``!Select`` intrinsic function, **except you can use this with negative indices to +select from the end of a list**. + +The argument for this resolver should be a list with two elements: (1) A numerical index and (2) a +list of items to select out of. If the index is negative, it will select from the end of the list. +For example, "-1" would select the last element and "-2" would select the second-to-last element. + +Example: + +.. code-block:: yaml + + sceptre_user_data: + # This selects the last element after you split the connection string on "/" + DatabaseName: !select + - -1 + - !split ["/", !stack_output my/database/stack.yaml::ConnectionString] + +split +~~~~~ + +This resolver will split a value on a given delimiter string. This is great when combining with the +``!select`` resolver. This function works the same as CloudFormation's ``!Split`` intrinsic function. + +Note: The return value of this resolver is a *list*, not a string. This will not work to set Stack +configurations that expect strings, but it WILL work to set Stack configurations that expect lists. + +The argument for this resolver should be a list with two elements: (1) The delimiter to split on and +(2) a string to split. Example: +.. code-block:: yaml + + notifications: !split + - ";" + - !stack_output my/sns/topics.yaml::SemicolonDelimitedArns + +.. _stack_attr_resolver: + +stack_attr +~~~~~~~~~~ + +This resolver resolves to the values of other fields on the same Stack Config or those +inherited from StackGroups in which the current Stack Config exists, even when those other fields are +also resolvers. + +To understand why this is useful, consider a stack's ``template_bucket_name``. This is usually set on +the highest level StackGroup Config. Normally, you could reference the template_bucket_name that was +set in an outer StackGroup Config with Jinja using ``{{template_bucket_name}}`` or, more explicitly, with +``{{stack_group_config.template_bucket_name}}``. + +However, if the value of ``template_bucket_name`` is set with a resolver, using Jinja won't work. +This is due to the :ref:`resolution_order` on a Stack Config. Jinja configs are rendered *before* +resolvers are constructed or resolved, so you can't resolve a resolver from a StackGroup Config via +Jinja. That's where !stack_attr is useful. It's a resolver that resolves to the value of another stack +attribute (which could be another resolver). + +.. code-block:: yaml + + template: + type: sam + path: path/from/my/cwd/template.yaml + # template_bucket_name could be set by a resolver in the StackGroup. + artifact_bucket_name: !stack_attr template_bucket_name + +The argument to this resolver is the full attribute "path" from the Stack Config. You can access +nested values in dicts and lists using "." to separate key/index segments. For example: + .. code-block:: yaml sceptre_user_data: - iam_policy: !file_contents /path/to/policy.json + key: + - "some random value" + - "the value we want to select" + + sceptre_role: !stack_output roles.yaml::RoleArn + + parameters: + # This will pass the value of "the value we want to select" for my_parameter + my_parameter: !stack_attr sceptre_user_data.key.1 + # You can also access the value of another resolvable property like this: + use_role: !stack_attr sceptre_role + stack_output ~~~~~~~~~~~~ @@ -78,10 +212,16 @@ Example: VpcIdParameter: !stack_output shared/vpc.yaml::VpcIdOutput Sceptre infers that the Stack to fetch the output value from is a dependency, -and builds that Stack before the current one. +adding that stack to the current stack's list of dependencies. This instructs +Sceptre to build that Stack before the current one. -This resolver will add a dependency for the Stack in which needs the output -from. +.. warning:: + Be careful when using the stack_output resolver that you do not create circular dependencies. + This is especially true when using this on StackGroup Configs to create configurations + to be inherited by all stacks in that group. If the `!stack_output` resolver would be "inherited" + from a StackGroup Config by the stack it references, this will lead to a circular dependency. + The correct way to work around this is to move that stack outside that StackGroup so that it + doesn't "inherit" that resolver. stack_output_external ~~~~~~~~~~~~~~~~~~~~~ @@ -107,6 +247,67 @@ Example: parameters: VpcIdParameter: !stack_output_external prj-network-vpc::VpcIdOutput prod + +sub +~~~ + +This resolver allows you to create a string using Python string format syntax. This functions as a +great way to combine together a number of resolver outputs into a single string. This functions +similarly to Cloudformation's ``!Sub`` intrinsic function. + +It should be noted that Jinja2 syntax is far more capable of interpolating values than this resolver, +so you should use Jinja2 if all you need is to interpolate raw values from environment variables, +variables from stack group configs, var files, and ``--var`` arguments. **The one thing that Jinja2 +interpolation can't do is interpolate resolver arguments into a string.** And that's what ``!sub`` +can do. For more information on why Jinja2 can't reference resolvers directly, see +:ref:`resolution_order`. + +The argument to this resolver should be a two-element list: (1) Is the format string, using +curly-brace templates to indicate variables, and (2) a dictionary where the keys are the format +string's variable names and the values are the variable values. + +Example: + +.. code-block:: yaml + + parameters: + ConnectionString: !sub + - "postgres://{username}:{password}@{hostname}:{port}/{database}" + # Notice how we're interpolating a username and database via Jinja2? Technically it's not + # necessary to pass them this way. They could be interpolated directly. But it might be + # easier to read this way if you pass them explicitly like this. See example below for the + # other way this can be done. + - username: {{ var.username }} + password: !ssm /my/ssm/password + hostname: !stack_output my/database/stack.yaml::HostName + port: !stack_output my/database/stack.yaml::Port + database: {{var.database}} + + +It's relevant to note that this functions similarly to the *more verbose* form of CloudFormation's +``!Sub`` intrinsic function, where you use a list argument and supply the interpolated values as a +second list item in a dictionary. **Important**: Sceptre's ``!sub`` resolver will not work without +a list argument. It does **not** directly reference variables without you directly passing them +in the second list item in its argument. + +You *can* combine Jinja2 syntax with this resolver if you want to interpolate in other variables +that Jinja2 has access to. + +Example: + +.. code-block:: yaml + + parameters: + ConnectionString: !sub + # Notice the double-curly braces. That's Jinja2 syntax. Jinja2 will render the username into + # the string even before the yaml is loaded. If you use Jinja2 to interpolate the value, then + # it's not a template string variable you need to pass in the second list item passed to + # !sub. + - "postgres://{{ var.username }}:{password}@{hostname}:{port}/{{ stack_group_config.database }}" + - password: !ssm /my/ssm/password + hostname: !stack_output my/database/stack.yaml::HostName + port: !stack_output my/database/stack.yaml::Port + Custom Resolvers ---------------- @@ -155,15 +356,23 @@ custom_resolver.py Parameters ---------- - argument: str - The argument of the resolver. + argument: Any + The argument of the resolver. This can be any value able to be defined in yaml. stack: sceptre.stack.Stack - The associated stack of the resolver. - + The associated stack of the resolver. This will normally be None when the resolver is + instantiated, but will be set before the resolver is resolved. """ - def __init__(self, *args, **kwargs): - super(CustomResolver, self).__init__(*args, **kwargs) + def __init__(self, argument, stack=None): + super(CustomResolver, self).__init__(argument, stack) + + def setup(self): + """ + Setup is invoked after the stack has been set on the resolver, whether or not the + resolver is ever resolved. + + Implement this method for any setup behavior you want (such as adding to stack dependencies). + """ def resolve(self): """ @@ -209,9 +418,87 @@ This resolver can be used in a Stack config file with the following syntax: .. code-block:: yaml - template_path: <...> + template: + path: <...> + type: <...> parameters: param1: ! +Calling AWS services in your custom resolver +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For details on calling AWS services or invoking AWS-related third party tools in your resolver, see +:ref:`using_connection_manager` + + +Resolver arguments +^^^^^^^^^^^^^^^^^^ +Resolver arguments can be a simple string or a complex data structure. You can even use +other resolvers in the arguments to resolvers! (Note: Other resolvers can only be passed in +arguments when they're passed in lists and dicts.) + +.. code-block:: yaml + + template: + path: <...> + type: <...> + parameters: + Param1: !ssm "/dev/DbPassword" + Param2: !ssm {"name": "/dev/DbPassword"} + Param3: !ssm + name: "/dev/DbPassword" + Param4: !ssm + name: !stack_output my/other/stack.yaml::MySsmParameterName + .. _Custom Resolvers: #custom-resolvers -.. _this is great place to start: https://docs.python.org/3/distributing/ \ No newline at end of file +.. _this is great place to start: https://docs.python.org/3/distributing/ + +Resolving to nothing +^^^^^^^^^^^^^^^^^^^^ +When a resolver returns ``None``, this means that it resolves to "nothing". For resolvers set for +single values (such as for ``template_bucket_name`` or ``cloudformation_service_role``), this just means the value is +``None`` and treated like those values aren't actually set. But for resolvers inside of containers +like lists or dicts, when they resolve to "nothing", that item gets completely removed from their +containing list or dict. + +This feature would be useful if you wanted to define a resolver that sometimes would resolve to be a +given stack parameter and sometimes would be not defined at all and use the template's default value +for that parameter. The resolver could just return `None` in those cases it wants to resolve to +nothing, similar to the AWS::NoValue pseudo-parameter that can be referenced in a CloudFormation +template. + +Resolver placeholders +^^^^^^^^^^^^^^^^^^^^^ +Resolvers (especially the !stack_output resolver) often express dependencies on other stacks and +their outputs. However, there are times when those stacks or outputs will not exist yet because they +have not yet been deployed. During normal deployment operations (using the ``launch``, ``create``, +``update``, and ``delete`` commands), Sceptre knows the correct order to resolve dependencies in and will +ensure that order is followed, so everything works as expected. + +But there are other commands that will not actually deploy dependencies of a stack config before +operating on that Stack Config. These commands include ``generate``, ``validate``, and ``diff``. +If you have used resolvers to reverence other stacks, it is possible that a resolver might not be able +to be resolved when performing that command's operations and will trigger an error. This is not likely +to happen when you have only used resolvers in a stack's ``parameters``, but it is much more likely +if you have used them in ``sceptre_user_data`` with a Jinja or Python template. At those times (and +only when a resolver cannot be resolved), a **best-attempt placeholder value** will be supplied in to +allow the command to proceed. Depending on how your template or Stack Config is configured, the +command may or may not actually succeed using that placeholder value. + +A few examples... + +* If you have a stack parameter referencing ``!stack_output other_stack.yaml::OutputName``, + and you run the ``diff`` command before other_stack.yaml has been deployed, the diff output will + show the value of that parameter to be ``"{ !StackOutput(other_stack.yaml::OutputName) }"``. +* If you have a ``sceptre_user_data`` value used in a Jinja template referencing + ``!stack_output other_stack.yaml::OutputName`` and you run the ``generate`` command, the generated + template will replace that value with ``"StackOutputotherstackyamlOutputName"``. This isn't as + "pretty" as the sort of placeholder used for stack parameters, but the use of sceptre_user_data is + broader, so it placeholder values can only be alphanumeric to reduce chances of it breaking the + template. +* Resolvable properties that are *always* used when performing template operations (like ``sceptre_role`` + and ``template_bucket_name``) will resolve to ``None`` and not be used for those operations if they + cannot be resolved. + +Any command that allows these placeholders can have them disabled with the ``--no-placeholders`` ClI +option. diff --git a/docs/_source/docs/stack_config.rst b/docs/_source/docs/stack_config.rst index 518e4390b..52ff8db03 100644 --- a/docs/_source/docs/stack_config.rst +++ b/docs/_source/docs/stack_config.rst @@ -1,8 +1,9 @@ Stack Config ============ -Stack config stores config related to a particular Stack, such as the path to -that Stack’s Template, and any parameters that Stack may require. +A Stack config stores configurations related to a particular Stack, such as the path to +that Stack’s Template, and any parameters that Stack may require. Many of these configuration keys +support resolvers and can be inherited from parent StackGroup configs. .. _stack_config-structure: @@ -12,21 +13,34 @@ Structure A Stack config file is a ``yaml`` object of key-value pairs configuring a particular Stack. The available keys are listed below. -- `template_path`_ *(required)* +- `template_path`_ or `template`_ *(required)* - `dependencies`_ *(optional)* - `hooks`_ *(optional)* +- `ignore`_ *(optional)* - `notifications`_ *(optional)* +- `obsolete`_ *(optional)* - `on_failure`_ *(optional)* +- `disable_rollback`_ *(optional)* - `parameters`_ *(optional)* - `protected`_ *(optional)* - `role_arn`_ *(optional)* +- `cloudformation_service_role`_ *(optional)* +- `iam_role`_ *(optional)* +- `sceptre_role`_ (*optional)* +- `iam_role_session_duration`_ *(optional)* +- `sceptre_role_session_duration`_ *(optional)* - `sceptre_user_data`_ *(optional)* - `stack_name`_ *(optional)* - `stack_tags`_ *(optional)* - `stack_timeout`_ *(optional)* -template_path - required +It is not possible to define both `template_path`_ and `template`_. If you do so, +you will receive an error when deploying the stack. + +template_path ~~~~~~~~~~~~~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: No The path to the CloudFormation, Jinja2 or Python template to build the Stack from. The path can either be absolute or relative to the Sceptre Directory. @@ -34,22 +48,110 @@ Sceptre treats the template as CloudFormation, Jinja2 or Python depending on the template’s file extension. Note that the template filename may be different from the Stack config filename. +.. warning:: + + This key is deprecated in favor of the `template`_ key. It will be removed in version 5.0.0. + +template +~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: No + +Configuration for a template handler. Template handlers can take in parameters +and resolve that to a CloudFormation template. This enables you to not only +load templates from disk, but also from third-party storage or AWS services. + +Example for loading from S3 bucket: + +.. code-block:: yaml + + template: + type: s3 + path: infra-templates/s3/v1/bucket.yaml + parameters: + : "value" + sceptre_user_data: + +It is possible to write your own template handlers should you need to. You +can find a list of currently supported template handlers and guidance for +developing your own in the :doc:`template_handlers` section. + dependencies ~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Appended to parent's dependencies A list of other Stacks in the environment that this Stack depends on. Note that if a Stack fetches an output value from another Stack using the ``stack_output`` resolver, that Stack is automatically added as a dependency, and that Stack need not be added as an explicit dependency. +.. warning:: + Be careful about how you structure dependencies. It is possible to create circular + dependencies accidentally, where multiple stacks depend on each other. Sceptre + will detect this and raise an error, blocking this sort of setup. You must be especially careful + when specifying ``dependencies`` on a StackGroup config. These dependencies will then be + "inherited" by every stack within that StackGroup. If one of those dependencies *inherits* that + list of dependencies, it will cause a circular dependency. If this happens, you can resolve the + situation by either (a) setting those ``dependencies`` on individual Stack Configs rather than the + the StackGroup Config, or (b) moving those dependency stacks outside of the StackGroup. + hooks ~~~~~ +* Resolvable: No (but you can use resolvers _in_ hook arguments!) +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set A list of arbitrary shell or Python commands or scripts to run. Find out more in the :doc:`hooks` section. +ignore +~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +This configuration should be set with a boolean value of ``True`` or ``False``. By default, this is +set to ``False`` on all stacks. + +``ignore`` determines how the stack should be handled when running ``sceptre launch``. A stack +marked with ``ignore: True`` will be completely ignored by the launch command. If the stack does NOT +exist on AWS, it won't be created. If it *DOES* exist, it will neither be updated nor deleted. + +You *can* mark a stack with ``ignore: True`` that other non-ignored stacks depend on, but the launch +will fail if dependent stacks require resources or outputs that don't exist because the stack has not been +launched. **Therefore, only ignore dependencies of other stacks if you are aware of the risks of +launch failure.** + +This setting can be especially useful when combined with Jinja logic to exclude certain stacks from +launch based upon conditional Jinja-based template logic. + +For Example: + +.. code-block:: yaml + + template: + path: "my/test/resources.yaml" + + # Configured this way, if the var "use_test_resources" is not true, the stack will not be launched + # and instead excluded from the launch. But if "use_test_resources" is true, the stack will be + # deployed along with the rest of the resources being deployed. + {% if not var.use_test_resources %} + ignore: True + {% endif %} + + +.. note:: + The ``ignore`` configuration **only** applies to the **launch** command. You can still run + ``create``, ``update``, or ``delete`` commands on a stack marked with ``ignore: True``; + these commands will ignore the ``ignore`` setting and act upon the stack the same as any other. + notifications ~~~~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set List of SNS topic ARNs to publish Stack related events to. A maximum of 5 ARNs can be specified per Stack. This configuration will be used by the ``create``, @@ -57,16 +159,81 @@ can be specified per Stack. This configuration will be used by the ``create``, can found under the relevant section in the `AWS CloudFormation API documentation`_. +.. _`obsolete`: + +obsolete +~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +This configuration should be set with a boolean value of ``True`` or ``False``. By default, this is +set to ``False`` on all stacks. + +The ``obsolete`` configuration should be used to mark stacks to be deleted via ``prune`` actions, +if they currently exist on AWS. (If they don't exist on AWS, pruning does nothing). + +There are two ways to prune obsolete stacks: + +1. ``sceptre prune`` will delete *all* obsolete stacks in the **project**. +2. ``sceptre launch --prune [command path]`` will delete all obsolete stacks in the command path + before continuing with the launch. + +In practice, the ``obsolete`` configuration operates identically to ``ignore`` with the extra prune +effects. When the ``launch`` command is invoked without the ``--prune`` flag, obsolete stacks will +be ignored and not launched, just as if ``ignore: True`` was on the Stack Config. + +**Important**: You cannot have non-obsolete stacks dependent upon obsolete stacks. Both the +``prune`` and ``launch --prune`` will reject such configurations and will not continue if this sort +of dependency structure is detected. Only obsolete stacks can depend on obsolete stacks. + +.. note:: + The ``obsolete`` configuration **only** applies to the **launch** and **prune** commands. You can + still run ``create``, ``update``, or ``delete`` commands on a stack marked with ``obsolete: True``; + these commands will ignore the ``obsolete`` setting and act upon the stack the same as any other. + on_failure ~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set This parameter describes the action taken by CloudFormation when a Stack fails to create. For more information and valid values see the `AWS Documentation`_. +Examples include: + +``on_failure: "DO_NOTHING"`` + +``on_failure: "ROLLBACK"`` + +``on_failure: "DELETE"`` + +disable_rollback +~~~~~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +This parameter describes the action taken by CloudFormation when a Stack fails +to create or update, default is False. This option can be set from the stack +config or from the Sceptre CLI commands to deploy stacks. The disable_rollback +CLI option (i.e. sceptre launch --disable-rollback) disables cloudformation +rollback globally for all stacks. This option overrides on_failure since +Cloudformation does not allow setting both on deployment. For more information +and valid values see the `AWS Documentation`_. + +Examples: + +``disable_rollback: "True"`` + parameters ~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set -.. container:: alert alert-danger +.. warning:: Sensitive data such as passwords or secret keys should not be stored in plaintext in Stack config files. Instead, they should be passed in from the @@ -77,10 +244,17 @@ A dictionary of key-value pairs to be supplied to a template as parameters. The keys must match up with the name of the parameter, and the value must be of the type as defined in the template. +.. note:: + Note that Boto3 throws an exception if parameters are supplied to a template that are not required by that template. Resolvers can be used to add functionality to this key. Find out more in the :doc:`resolvers` section. +.. warning:: + + In case the same parameter key is supplied more than once, the last + definition silently overrides the earlier definitions. + A parameter can be specified either as a single value/resolver or a list of values/resolvers. Lists of values/resolvers will be formatted into an AWS compatible comma separated string e.g. \ ``value1,value2,value3``. Lists can @@ -115,11 +289,14 @@ Example: - "subnet-87654321" security_group_ids: - "sg-12345678" - - !stack_output security-groups::BaseSecurityGroupId + - !stack_output security-groups.yaml::BaseSecurityGroupId - !file_contents /file/with/security_group_id.txt protected ~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set Stack protection against execution of the following commands: @@ -134,12 +311,87 @@ throw an error. role_arn ~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +.. warning:: + This field is deprecated as of v4.0.0 and will be removed in v5.0.0. It has been renamed to + `cloudformation_service_role`_ as a clearer name for its purpose. + +cloudformation_service_role +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +The ARN of a `CloudFormation Service Role`_ that is assumed by *CloudFormation* (not Sceptre) +to create, update or delete resources. For more information on this configuration, its implications, +and its uses see :ref:`Sceptre and IAM: cloudformation_service_role `. + +iam_role +~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +.. warning:: + This field is deprecated as of v4.0.0 and will be removed in v5.0.0. It has been renamed to + `sceptre_role`_ as a clearer name for its purpose. + +sceptre_role +~~~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +This is the IAM Role ARN that **Sceptre** should *assume* using AWS STS when executing any actions +on the Stack. + +This is different from the ``cloudformation_service_role`` option, which sets a CloudFormation +service role for the stack. The ``sceptre_role`` configuration does not configure anything on the +stack itself. + +.. warning:: + + If you set the value of ``sceptre_role`` with ``!stack_output``, that ``sceptre_role`` + will not actually be used to obtain the stack_output, but it *WILL* be used for all subsequent stack + actions. Therefore, it is important that the user executing the stack action have permissions to get + stack outputs for the stack outputting the ``sceptre_role``. + +For more information on this configuration, its implications, and its uses, see +:ref:`Sceptre and IAM: sceptre_role `. + +iam_role_session_duration +~~~~~~~~~~~~~~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +.. warning:: + This field is deprecated as of v4.0.0 and will be removed in v5.0.0. It has been renamed to + `sceptre_role_session_duration`_ as a clearer name for its purpose. + +sceptre_role_session_duration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set -The ARN of a `CloudFormation Service Role`_ that is assumed by CloudFormation -to create, update or delete resources. +This is the session duration when **Sceptre** *assumes* the **sceptre_role** IAM Role using AWS STS when +executing any actions on the Stack. + +.. warning:: + + If you set the value of ``sceptre_role_session_duration`` to a number that *GREATER* than 3600, you + will need to make sure that the ``sceptre_role`` has a configuration of ``MaxSessionDuration``, and + its value is *GREATER* than or equal to the value of ``sceptre_role_session_duration``. sceptre_user_data ~~~~~~~~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set Represents data to be passed to the ``sceptre_handler(sceptre_user_data)`` function in Python templates or accessible under ``sceptre_user_data`` variable @@ -147,6 +399,8 @@ key within Jinja2 templates. stack_name ~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: No A custom name to use instead of the Sceptre default. @@ -163,7 +417,7 @@ e.g: .. code-block:: yaml parameters: - VpcID: !stack_output_external .yaml::VpcID + VpcID: !stack_output_external ::VpcID dependencies: - / @@ -174,17 +428,23 @@ referring to is in a different AWS account or region. .. code-block:: yaml parameters: - VpcID: !stack_output_external .yaml::VpcID my-aws-prod-profile + VpcID: !stack_output_external ::VpcID my-aws-prod-profile dependencies: - / stack_tags ~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set A dictionary of `CloudFormation Tags`_ to be applied to the Stack. stack_timeout ~~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set A timeout in minutes before considering the Stack deployment as failed. After the specified timeout, the Stack will be rolled back. Specifiyng zero, as well @@ -216,7 +476,7 @@ when using templating. .. code-block:: yaml parameters: - sceptre-project-code: {{ stack_group_config.sceptre-project-code }} + sceptre-project-code: {{ stack_group_config.project-code }} Environment Variables --------------------- @@ -247,19 +507,99 @@ Syntax: When compiled, ``sceptre_user_data`` would be the dictionary ``{"iam_policy_file": "/path/to/policy.json"}``. +.. _resolution_order: + +Resolution order of values +-------------------------- + +Stack Configs allow you to pull together values from a variety of sources to configure a +CloudFormation stack. These values are retrieved and applied in phases. Understanding these phases can +be very helpful when designing your Stack Configs. + +When launching a stack (or performing other stack actions), values are gathered and accessed in this +order: + +1. User variables (from ``--var`` and ``--var-file`` arguments) are gathered when the CLI first runs. +2. StackGroup Configs are read from the highest level downward, rendered with Jinja and then loaded + into yaml. The key/value pairs from these configs are layered on top of each other, with more nested + configs overriding higher-level ones. These key/value pairs will be "inherited" by the Stack + Config. These variables are made available when rendering a StackGroup Config: + + * User variables (via ``{{ var }}``) + * Environment variables (via ``{{ environment_variable }}``) + * StackGroup configurations from *higher* level StackGroup Configs are available by name. Note: + more nested configuration values will overshadow higher-level ones by the same key. + +3. With the layered StackGroup Config variables, the Stack Config file will be read and then rendered + with Jinja. These variables are made available when the Stack Config is being rendered with Jinja: + + * User variables (via ``{{ var }}``) + * Environment variables (via ``{{ environment_variable }}``) + * All StackGroup configurations are available by name directly as well as via ``{{ stack_group_config }}`` + + **Important:** If any StackGroup configuration values were set with resolvers, accessing them via + Jinja will not resolve them, since resolvers require a Stack object, which has not yet been + assembled yet. **Resolvers will not be accessible until a later phase.** +4. Once rendered via Jinja into a string, the Stack Config will be loaded into yaml. This is when the + resolver instances on the Stack config will be **constructed** (*not* resolved). +5. The Stack instance will be constructed with the key/value pairs from the loaded yaml layered on + top of the key/value pairs from the StackGroup configurations. This is when all resolver instances, + both those inherited from StackGroup Configs and those from the present Stack Config, will be + connected to the Stack instance and thus *ready* to be resolved. +6. The first time a resolvable configuration is *accessed* is when the resolver(s) at that + configuration will be resolved and replaced with their resolved value. This is normally done at + the very last moment, right when it is needed (and not before). + +"Render Time" vs. "Resolve Time" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A common point of confusion tends to be around the distinction between **"render time"** (phase 3, when +Jinja logic is applied) and **"resolve time"** (phase 6, when resolvers are resolved). You cannot use +a resolver via Jinja during "render time", since the resolver won't exist or be ready to use yet. You can, +however, use Jinja logic to indicate *whether*, *which*, or *how* a resolver is configured. You can +also use resolvers like ``!sub`` to interpolate resolved values when Jinja isn't available. + +For example, you **can** do something like this: + +.. code-block:: yaml + + parameters: + {% if var.use_my_parameter %} + my_parameter: !stack_output {{ var.stack_name }}::{{ var.output_name }} + {% endif %} + # !sub will let you combine outputs of multiple resolvers into a single string + my_combined_parameter: !sub + - "{fist_part} - {second_part}" + - first_part: !stack_output my/stack/name.yaml::Output + - second_part: {{ var.second_part }} + +Accessing resolved values in other fields +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes you might want to reference the resolved value of one field in another field. Since you cannot +use Jinja to access resolved values, there is another way to this. The :ref:`stack_attr_resolver` +resolver is meant for addressing just this need. It's a resolver that will resolve to the value of +another Stack Config field value. See the linked documentation for more details on that resolver and +its use. + + Examples -------- .. code-block:: yaml - template_path: templates/example.py + template: + path: templates/example.py + type: file parameters: param_1: value_1 param_2: value_2 .. code-block:: yaml - template_path: example.yaml + template: + path: templates/example.yaml + type: file dependencies: - dev/vpc.yaml hooks: @@ -272,8 +612,8 @@ Examples - !cmd "mkdir example" - !cmd "touch example.txt" parameters: - param_1: !stack_output stack_name::output_name - param_2: !stack_output_external full_stack_name.yaml::output_name + param_1: !stack_output stack_name.yaml::output_name + param_2: !stack_output_external full_stack_name::output_name param_3: !environment_variable VALUE_3 param_4: {{ var.value4 }} @@ -288,18 +628,20 @@ Examples tag_1: value_1 tag_2: value_2 -.. _template_path: #template_path +.. _template_path: #template-path +.. _template: #template .. _dependencies: #dependencies .. _hooks: #hooks .. _notifications: #notifications -.. _on_failure: #on_failure +.. _on_failure: #on-failure +.. _disable_rollback: #disable-rollback .. _parameters: #parameters .. _protected: #protected -.. _role_arn: #role_arn -.. _sceptre_user_data: #sceptre_user_data -.. _stack_name: #stack_name -.. _stack_tags: #stack_tags -.. _stack_timeout: #stack_timeout +.. _role_arn: #role-arn +.. _sceptre_user_data: #sceptre-user-data +.. _stack_name: #stack-name +.. _stack_tags: #stack-tags +.. _stack_timeout: #stack-timeout .. _AWS CloudFormation API documentation: http://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html .. _AWS Documentation: http://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html .. _CloudFormation Service Role: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-iam-servicerole.html diff --git a/docs/_source/docs/stack_group_config.rst b/docs/_source/docs/stack_group_config.rst index 4797e60d4..3da527e4b 100644 --- a/docs/_source/docs/stack_group_config.rst +++ b/docs/_source/docs/stack_group_config.rst @@ -19,30 +19,45 @@ Sceptre. The available keys are listed below. - `required_version`_ *(optional)* - `template_bucket_name`_ *(optional)* - `template_key_prefix`_ *(optional)* +- `j2_environment`_ *(optional)* +- `http_template_handler`_ *(optional)* Sceptre will only check for and uses the above keys in StackGroup config files and are directly accessible from Stack(). Any other keys added by the user are -made available via ``stack_group_confg`` attribute on ``Stack()``. +made available via ``stack_group_config`` attribute on ``Stack()``. profile ~~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child -The name of the profile as defined in ``~/.aws/config`` and -``~/.aws/credentials``. +The name of the profile as defined in ``~/.aws/config`` and ``~/.aws/credentials``. Use the +`aws configure --profile ` command form the AWS CLI to add profiles to these files. + +For more information on this configuration, its implications, and its uses, see +:ref:`Sceptre and IAM: profile `. + +Reference: `AWS_CLI_Configure`_ project_code ~~~~~~~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child A string which is prepended to the Stack names of all Stacks built by Sceptre. region ~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child The AWS region to build Stacks in. Sceptre should work in any `region which supports CloudFormation`_. template_bucket_name ~~~~~~~~~~~~~~~~~~~~ +* Resolvable: Yes +* Inheritance strategy: Overrides parent if set by child The name of an S3 bucket to upload CloudFormation Templates to. Note that S3 bucket names must be globally unique. If the bucket does not exist, Sceptre @@ -53,8 +68,36 @@ supplies the template to Boto3 via the ``TemplateBody`` argument. Templates supplied in this way have a lower maximum length, so using the ``template_bucket_name`` parameter is recommended. +.. warning:: + + If you resolve ``template_bucket_name`` using the ``!stack_output`` + resolver on a StackGroup, the stack that outputs that bucket name *cannot* be + defined in that StackGroup. Otherwise, a circular dependency will exist and Sceptre + will raise an error when attempting any Stack action. There are two ways to avoid this situation: + + 1. Set the ``template_bucket_name`` to ``!no_value`` in on the StackConfig that creates your + template bucket. This will override the inherited value to prevent them from having + dependencies on themselves. + 2. Define all your project stacks inside a StackGroup and then your template bucket + stack *outside* that StackGroup. Here's an example project structure for something like + this: + + .. code-block:: yaml + + config/ + - config.yaml # This is the StackGroup Config for your whole project. + - template-bucket.yaml # The template for this stack outputs the bucket name + - project/ # You can put all your other stacks in this StackGroup + - config.yaml # In this StackGroup Config is... + # template_bucket_name: !stack_output template-bucket.yaml::BucketName + - vpc.yaml # Put all your other project stacks inside project/ + - other-stack.yaml + + template_key_prefix ~~~~~~~~~~~~~~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child A string which is prefixed onto the key used to store templates uploaded to S3. Templates are stored using the key: @@ -71,13 +114,44 @@ Extension can be ``json`` or ``yaml``. Note that if ``template_bucket_name`` is not supplied, this parameter is ignored. +j2_environment +~~~~~~~~~~~~~~ +* Resolvable: No +* Inheritance strategy: Child configs will be merged with parent configs + +A dictionary that is combined with the default jinja2 environment. +It's converted to keyword arguments then passed to [jinja2.Environment](https://jinja.palletsprojects.com/en/2.11.x/api/#jinja2.Environment). +This will impact the templating of stacks by modifying the behavior of jinja. + +.. code-block:: yaml + + j2_environment: + extensions: + - jinja2.ext.i18n + - jinja2.ext.do + lstrip_blocks: True + trim_blocks: True + newline_sequence: \n + +http_template_handler +~~~~~~~~~~~~~~~~~~~~~ + +Options passed to the `http template handler`_. + * retries - The number of retry attempts (default is 5) + * timeout - The timeout for the session in seconds (default is 5) + +.. code-block:: yaml + + http_template_handler: + retries: 10 + timeout: 20 + require_version ~~~~~~~~~~~~~~~ A `PEP 440`_ compatible version specifier. If the Sceptre version does not fall within the given version requirement it will abort. - .. _stack_group_config_cascading_config: Cascading Config @@ -100,7 +174,8 @@ General configurations should be defined at a high level, and more specific configurations should be defined at a lower directory level. YAML files that define configuration settings with conflicting keys, the child -configuration file will take precedence. +configuration file will usually take precedence (see the specific config keys as documented +for the inheritance strategy employed). In the above directory structure, ``config/config.yaml`` will be read in first, followed by ``config/account-1/config.yaml``, followed by @@ -110,6 +185,53 @@ For example, if you wanted the ``dev`` StackGroup to build to a different region, this setting could be specified in the ``config/dev/config.yaml`` file, and would only be applied to builds in the ``dev`` StackGroup. +.. _setting_dependencies_for_stack_groups: + +Setting Dependencies for StackGroups +------------------------------------ +There are a few pieces of AWS infrastructure that Sceptre can (optionally) use to support the needs +and concerns of the project. These include: + +* The S3 bucket where templates are uploaded to and then referenced from for stack actions (i.e. the + ``template_bucket_name`` config key). +* The CloudFormation service role added to the stack(s) that CloudFormation uses to execute stack + actions (i.e. the ``cloudformation_service_role`` config key). +* The role that Sceptre will assume to execute stack actions (i.e. the ``sceptre_role`` config key). +* SNS topics that cloudformation will notify with the results of stack actions (i.e. the + ``notifications`` config key). + +These sorts of dependencies CAN be defined in Sceptre and added at the StackGroup level, referenced +using ``!stack_output``. Doing so will make it so that every stack in the StackGroup will have those +dependencies and get those values from Sceptre-managed stacks. + +Beyond the above mentioned config keys, it is possible to set the ``dependencies`` config key in a +StackGroup config to be inherited by all Stack configs in that group. All dependencies in child +stacks will be added to their inherited StackGroup dependencies, so be careful how you structure +dependencies. + +.. warning:: + + You might have already considered that this might cause a circular dependency for those + dependency stacks, the ones that output the template bucket name, role arn, sceptre_role, or topic arns. + In order to avoid the circular dependency issue, you can either: + + 1. Set the value of those configurations to ``!no_value`` in the actual stacks that define those + items so they don't inherit a dependency on themselves. + 2. Define those stacks *outside* the StackGroup you reference them in. Here's an example project + structure that would support doing this: + + .. code-block:: yaml + + config/ + - config.yaml # This is the StackGroup Config for your whole project. + - sceptre-dependencies.yaml # This stack defines your template bucket, iam role, topics, etc... + - project/ # You can put all your other stacks in this StackGroup + - config.yaml # In this StackGroup Config you can use !stack_output to + # reference outputs from sceptre-dependencies.yaml. + - vpc.yaml # Put all your other project stacks inside project/ + - other-stack.yaml + + .. _stack_group_config_templating: Templating @@ -188,6 +310,35 @@ Will result in the following variables being available to the jinja templating: profile: prod project_code: api +Note that by default, dictionaries are not merged. If the variable appearing in +the last variable file is a dictionary, and the same variable is defined in an +earlier variable file, that whole dictionary will be overwritten. For example, +this would not work as intended: + +.. code-block:: yaml + + # default.yaml + tags: {"Env": "dev", "Project": "Widget"} + +.. code-block:: yaml + + # prod.yaml + tags: {"Env": "prod"} + +Rather, the final dictionary would only contain the ``Env`` key. + +By using the ``--merge-vars`` option, these tags can be merged as intended: + +.. code-block:: text + + sceptre --merge-vars --var-file=default.yaml --var-file=prod.yaml --var region=us-east-1 + +This will result in the following: + +.. code-block:: yaml + + tags: {"Env": "prod", "Project": "Widget"} + For command line flags, Sceptre splits the string on the first equals sign “=”, and sets the key to be the first substring, and the value to be the second. Due to the large number of possible user inputs, no error checking is performed on @@ -261,3 +412,5 @@ Examples .. _template_key_prefix: #template_key_prefix .. _region which supports CloudFormation: http://docs.aws.amazon.com/general/latest/gr/rande.html#cfn_region .. _PEP 440: https://www.python.org/dev/peps/pep-0440/#version-specifiers +.. _AWS_CLI_Configure: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html +.. _http template handler: template_handlers.html#http diff --git a/docs/_source/docs/template_handlers.rst b/docs/_source/docs/template_handlers.rst new file mode 100644 index 000000000..cb77f9412 --- /dev/null +++ b/docs/_source/docs/template_handlers.rst @@ -0,0 +1,228 @@ +Template Handlers +================= + +Template handlers can be used to resolve a ``template`` config block to a CloudFormation template. This can be used to +fetch templates from S3, for example. Users can create their own template handlers to easily add support for other +template loading mechanisms. See `Custom Template Handlers`_ for more information. + +.. warning:: + + The ``template_path`` key is deprecated in favor of the ``template`` key. + +Available Template Handlers +--------------------------- + +file +~~~~~~~~~~~~~~~~~~~~ + +Loads a template from the local file system. This handler supports templates with .json, .yaml, .template, .j2 +and .py extensions. This is the default template handler type, specifying the ``file`` type is optional. + +For backwards compatability, when a ``template_path`` key is specified in the Stack config, it is wired to +use the ``file`` template handler. + +Syntax: + +.. code-block:: yaml + + template: + type: file + path: + +Example: + +.. code-block:: yaml + + template: + path: storage/bucket.yaml + +.. note:: + + The ``path`` property can contain an absolute or relative path to the template file. + This handler assumes a relative path to be from sceptre_project_dir/templates + + +s3 +~~~~~~~~~~~~~ + +Downloads a template from an S3 bucket. The bucket is accessed with the same credentials that is used to run sceptre. +This handler supports templates with .json, .yaml, .template, .j2 and .py extensions. + +Syntax: + +.. code-block:: yaml + + template: + type: s3 + path: / + +Example: + +.. code-block:: yaml + + template: + type: s3 + path: infra-templates/v1/storage/bucket.yaml + +http +~~~~~~~~~~~~~ + +Downloads a template from a url on the web. By default, this handler will attempt to download +templates with 5 retries and a download timeout of 5 seconds. The default retry and timeout +options can be overridden by setting the `http_template_handler key`_ in the stack group config +file. + +Syntax: + +.. code-block:: yaml + + template: + type: http + url: + +Example: + +.. code-block:: yaml + + template: + type: http + url: https://raw.githubusercontent.com/acme/infra-templates/v1/storage/bucket.yaml + + + +Custom Template Handlers +------------------------ + +If you need to load templates from a different source, you can write your own template handler. + +A template handler is a Python class which inherits from abstract base class ``TemplateHandler`` found in the +``sceptre.template_handlers`` module. + +To have Sceptre validate that the ``template`` block specified in the Stack config is correct, template handlers +should provide a JSON schema with the required and optional properties. The ``schema()`` method should be +implemented and return a Python dictionary with the schema. For examples of JSON schemas in Python, please see +the documentation of the `jsonschema library`_. + +Template handlers get access to the ``template`` block parameters, ``sceptre_user_data`` and ``connection_manager``. +These properties are available on ``self``. Using ``connection_manager``, template handlers can call AWS endpoints +to perform actions or fetch templates. These correspond to the AWS Python SDK (see Boto3_). For example: + +.. code-block:: python + + self.connection_manager.call( + service="s3", + command="get_object", + kwargs={ + "Bucket": bucket, + "Key": key + } + ) + +Sceptre uses the ``sceptre.template_handlers`` entry point to load template handlers. They can be written anywhere and +are installed as Python packages. + +Example +~~~~~~~ + +The following Python module template can be copied and used: + +.. code-block:: text + + custom_template_handler + ├── custom_template_handler.py + └── setup.py + +The following Python module template can be copied and used: + +custom_template_handler.py +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + from sceptre.template_handlers import TemplateHandler + + + class CustomTemplateHandler(TemplateHandler): + """ + The following instance attributes are inherited from the parent class TemplateHandler. + + Parameters + ---------- + name: str + The name of the template. Corresponds to the name of the Stack this template belongs to. + handler_config: dict + Configuration of the template handler. All properties except for `type` are available. + sceptre_user_data: dict + Sceptre user data defined in the Stack config + connection_manager: sceptre.connection_manager.ConnectionManager + Connection manager that can be used to call AWS APIs + """ + + def __init__(self, *args, **kwargs): + super(CustomTemplateHandler, self).__init__(*args, **kwargs) + + def schema(self): + """ + Return a JSON schema of the properties that this template handler requires. + For help filling this, see https://github.com/Julian/jsonschema + """ + return { + "type": "object", + "properties": {}, + "required": [] + } + + def handle(self): + """ + `handle` should return a CloudFormation template string or bytes. If the return + value is a byte array, UTF-8 encoding is assumed. + + To use instance attribute self.. See the class-level docs for a + list of attributes that are inherited. + + Returns + ------- + str|bytes + CloudFormation template + """ + return "" + + +setup.py +^^^^^^^^ + +.. code-block:: python + + from setuptools import setup + + setup( + name='', + py_modules=[''], + entry_points={ + 'sceptre.template_handlers': [ + ' = :CustomTemplateHandler', + ], + } + ) + +Then install using ``python setup.py install`` or ``pip install .`` commands. + +This template handler can be used in a Stack config file with the following syntax. Any properties you put in the +``template`` block will be passed to the template handler in the ``self.handler_config`` dictionary. + +.. code-block:: yaml + + template: + type: + : + +.. _jsonschema library: https://github.com/Julian/jsonschema +.. _Custom Template Handlers: #custom-template-handlers +.. _Boto3: https://aws.amazon.com/sdk-for-python/ +.. _http_template_handler key: stack_group_config.html#http-template-handler + +Calling AWS services in your custom template_handler +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For details on calling AWS services or invoking AWS-related third party tools in your +template handler, see :ref:`using_connection_manager` diff --git a/docs/_source/docs/templates.rst b/docs/_source/docs/templates.rst index a46538767..88a99eb6c 100644 --- a/docs/_source/docs/templates.rst +++ b/docs/_source/docs/templates.rst @@ -31,6 +31,121 @@ Template. Sceptre User Data is accessible within Templates as ``sceptre_user_data`` accesses the ``sceptre_user_data`` key in the Stack Config file. + +Example +~~~~~~~ + +While rendering templates with Jinja2, some characters aren't supported by AWS. A workaround for this is to use Jinja to replace `-`, `.` and `_` with other characters to ensure that we produce a valid template. With this we had some plasticity to the template and it becomes easier to add/remove entries on a particular stack. + +With this templating some problems will arise, such empty strings as parameters. In the following example you can find a work around for this issues. + +This represents a stack to deploy route53 records, it's only showing CNAME and ALIAS records to not get too large. + +Stack: + +.. code-block:: yaml + + template + path: templates/dns-extras.j2 + type: file + dependencies: + - prod/route53/domain-zone.yaml + parameters: + DomainName: "example.com"!stack_output prod/route53/example-com-zone.yaml::FullDomainName + Zone: !stack_output prod/route53/example-com-zone.yaml::HostedZoneID + sceptre_user_data: + CNAMErecords: + - record: "example01" + address: "example01.otherdomain.com." + ttl: 600 + - record: "example02" + address: "example01.otherdomain.com." + ttl: 600 + ALIASrecords: + - record: "" + hostedzoneid: "ZYOURZONEIDDDD" + dnsnamealias: "ELB07-Sites-000000000.us-east-1.elb.amazonaws.com" + ttl: 600 + - record: "www" + hostedzoneid: "Z32O12XQLNTSW2" + dnsnamealias: "ELB07-Sit es-000000000.us-east-1.elb.amazonaws.com" + ttl: 600 + + +Template `dns-extras.j2`: + +.. code-block:: jinja + + AWSTemplateFormatVersion: '2010-09-09' + Description: 'Add Route53 - CNAME and ALIAS records' + Parameters: + DomainName: + Type: String + Default: example.net + Zone: + Type: String + {% if sceptre_user_data.CNAMErecords is defined %}{% for rule in sceptre_user_data.CNAMErecords %} + {{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}cnamerecord: + Type: String + Default: "{{rule.record}}"{% endfor %}{% endif %} + {% if sceptre_user_data.ALIASrecords is defined %}{% for rule in sceptre_user_data.ALIASrecords %} + {{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}aliasrecord: + Type: String + Default: "{{rule.record}}" + {{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}aliasvalue: + Type: String + Default: "{{rule.dnsnamealias}}" + {{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}aliaszoneid: + Type: String + Default: "{{rule.hostedzoneid}}" + {% endfor %}{% endif %} + Resources: + {% if sceptre_user_data.CNAMErecords is defined %}{% for rule in sceptre_user_data.CNAMErecords %}add{{ rule.record | replace("-","d")|replace("_","s")|replace('.',"p")}}cnamerecord: + {% set record = rule.record %} + Type: 'AWS::Route53::RecordSet' + Properties: + Name: !Join + - "" + - [ !Sub '${ {{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}cnamerecord }','.', !Ref DomainName, '.'] + HostedZoneId: !Sub '${Zone}' + Type: CNAME + TTL: {{ rule.ttl }} + ResourceRecords: + - {{ rule.address }} + {% endfor %}{% endif %} + {% if sceptre_user_data.ALIASrecords is defined %}{% for rule in sceptre_user_data.ALIASrecords %} + {% set entry = rule.record |replace("-","d")|replace("_","s")|replace('.',"p")%}add{{entry}}aliasrecord: + Type: AWS::Route53::RecordSet + Properties: + {% if rule.record == "" %} + Name: !Ref DomainName + {% else %} + Name: !Join + - "" + - [ !Sub '${ {{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}aliasrecord }','.', !Ref DomainName, '.'] + {% endif %} + Type: A + HostedZoneId: !Ref Zone + AliasTarget: + DNSName: "{{ rule.dnsnamealias }}" + HostedZoneId: "{{ rule.hostedzoneid }}" + {% endfor %}{% endif %} + Outputs: + {% if sceptre_user_data.CNAMErecords is defined %}{% for rule in sceptre_user_data.CNAMErecords %}add{{ rule.record | replace("-","d")|replace("_","s")|replace('.',"p")}}cnamerecord: + Value: !Ref 'add{{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}cnamerecord' + Description: '{{ rule.address }}' + {% endfor %}{% endif %} + {% if sceptre_user_data.ALIASrecords is defined %}{% for rule in sceptre_user_data.ALIASrecords %}add{{ rule.record | replace("-","d")|replace("_","s")|replace('.',"p")}}aliasrecord: + Value: !Ref 'add{{ rule.record |replace("-","d")|replace("_","s")|replace('.',"p")}}aliasrecord' + Description: '{{ rule.dnsnamealias }}' + {% endfor %}{% endif %} + StackName: + Description: 'Stack name.' + Value: !Sub '${AWS::StackName}' + Export: + Name: !Sub '${AWS::StackName}' + + Python ------ @@ -43,6 +158,9 @@ Config file, Sceptre passes an empty ``dict``. Example ~~~~~~~ +Troposphere +^^^^^^^^^^^ + This example is using `troposphere`_ to generate CloudFormation Template as a `json` string. @@ -63,4 +181,30 @@ to generate CloudFormation Template as a `json` string. def sceptre_handler(sceptre_user_data): return vpc(sceptre_user_data) +.. note:: + To generate templates using Troposphere you must install the + Troposphere library by running ``pip install sceptre[troposphere]`` + .. _troposphere: https://github.com/cloudtools/troposphere/ + +AWS CDK +^^^^^^^ + +AWS CDK can be used to programmatically generate templates for your stacks and then Sceptre can +deploy those stacks. Taking this approach leverages the best capabilities of CDK (simple template +generation with sane defaults) and the best capabilities of Sceptre ("wiring together" stacks and +deploying them as consistent environments, leveraging hooks and resolvers to dynamically connect them). + +In order to use CDK with Sceptre, you need to install the `sceptre_cdk_handler`_ package. You can find +documentation and examples on how to use it on the `github repository`_. + +.. _sceptre_cdk_handler: https://pypi.org/project/sceptre-cdk-handler/ +.. _github repository: https://github.com/sceptre/sceptre-cdk-handler + +AWS SAM +^^^^^^^ +There is now a SAM Template Handler that lets you incorporate SAM templates into environments that +are managed and deployed using Sceptre. For more information on how to install and use SAM in your +Sceptre project, see the `sceptre-sam-handler`_ page on PyPI. + +.. _sceptre-sam-handler: https://pypi.org/project/sceptre-sam-handler/ diff --git a/docs/_source/docs/terminology.rst b/docs/_source/docs/terminology.rst index cdab65d22..7e2df5d6e 100644 --- a/docs/_source/docs/terminology.rst +++ b/docs/_source/docs/terminology.rst @@ -72,7 +72,7 @@ SceptrePlanExecutor You won’t be able to interact with the ``SceptrePlanExecutor`` directly but this part of the code is responsible for taking a ``SceptrePlan`` and ensuring all commands on every stack, are executed in the correct order, concurrently. -The executor algorithm focuses on correctness over maximal concurrency. It know +The executor algorithm focuses on correctness over maximal concurrency. It knows what to execute and when based on a ``StackGraph`` which is created when a ``SceptrePlan`` is created. diff --git a/docs/_source/index.rst b/docs/_source/index.rst index c7631fd0c..f4ebc302d 100644 --- a/docs/_source/index.rst +++ b/docs/_source/index.rst @@ -16,9 +16,11 @@ docs/stack_group_config.rst docs/stack_config.rst docs/templates.rst + docs/template_handlers.rst docs/hooks.rst docs/resolvers.rst docs/architecture.rst + docs/permissions.rst .. toctree:: :maxdepth: 3 diff --git a/docs/docs_requirements.txt b/docs/docs_requirements.txt index ec0aa412e..ea2be46cb 100644 --- a/docs/docs_requirements.txt +++ b/docs/docs_requirements.txt @@ -1,3 +1,4 @@ -sphinx>=1.8.2<2.0 -sphinx_click>=1.4.1<=3.0 -sphinx_rtd_theme==0.4.3 +Sphinx>=1.6.5,<5.0.0 +sphinx-click>=2.0.1,<4.0.0 +sphinx_rtd_theme==0.5.2 +docutils<0.17 # temporary fix for sphinx-rtd-theme==0.5.2, it depends on docutils<0.17 diff --git a/integration-tests/__ini__.py b/integration-tests/__ini__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/environment.py b/integration-tests/environment.py index e1de983c0..d3e793b4e 100644 --- a/integration-tests/environment.py +++ b/integration-tests/environment.py @@ -3,16 +3,19 @@ import uuid import yaml import boto3 +import string +import random def before_all(context): + random_str = "".join(random.choices(string.ascii_lowercase + string.digits, k=16)) + context.TEST_ARTIFACT_BUCKET_NAME = f"sceptre-test-artifacts-{random_str}" + context.region = boto3.session.Session().region_name context.uuid = uuid.uuid1().hex - context.project_code = "sceptre-integration-tests-{0}".format( - context.uuid - ) + context.project_code = "sceptre-integration-tests-{0}".format(context.uuid) sts = boto3.client("sts") - account_number = sts.get_caller_identity()['Account'] + account_number = sts.get_caller_identity()["Account"] context.bucket_name = "sceptre-integration-tests-templates-{}".format( account_number ) @@ -21,21 +24,9 @@ def before_all(context): os.getcwd(), "integration-tests", "sceptre-project" ) update_config(context) - context.cloudformation = boto3.resource('cloudformation') + context.cloudformation = boto3.resource("cloudformation") context.client = boto3.client("cloudformation") - config_path = os.path.join( - context.sceptre_dir, "config", "9/B" + ".yaml" - ) - - with open(config_path, "r") as file: - file_data = file.read() - - file_data = file_data.replace("{project_code}", context.project_code) - - with open(config_path, "w") as file: - file.write(file_data) - def before_scenario(context, scenario): os.environ.pop("AWS_REGION", None) @@ -46,29 +37,47 @@ def before_scenario(context, scenario): def update_config(context): - config_path = os.path.join( - context.sceptre_dir, "config", "config.yaml" - ) + config_path = os.path.join(context.sceptre_dir, "config", "config.yaml") with open(config_path) as config_file: stack_group_config = yaml.safe_load(config_file) stack_group_config["template_bucket_name"] = context.bucket_name stack_group_config["project_code"] = context.project_code - with open(config_path, 'w') as config_file: - yaml.safe_dump( - stack_group_config, config_file, default_flow_style=False - ) + with open(config_path, "w") as config_file: + yaml.safe_dump(stack_group_config, config_file, default_flow_style=False) def after_all(context): response = context.client.describe_stacks() for stack in response["Stacks"]: if stack["StackName"].startswith(context.project_code): - context.client.delete_stack( - StackName=stack["StackName"] - ) + context.client.delete_stack(StackName=stack["StackName"]) time.sleep(2) context.project_code = "sceptre-integration-tests" context.bucket_name = "sceptre-integration-tests-templates" update_config(context) + + +def before_feature(context, feature): + """ + Create a test bucket with a unique name and upload test artifact to the bucket + for the S3 template handler to reference + """ + if "s3-template-handler" in feature.tags: + bucket = boto3.resource("s3").Bucket(context.TEST_ARTIFACT_BUCKET_NAME) + if bucket.creation_date is None: + bucket.create( + CreateBucketConfiguration={"LocationConstraint": context.region} + ) + + +def after_feature(context, feature): + """ + Do a full cleanup of the test artifacts and the test bucket + """ + if "s3-template-handler" in feature.tags: + bucket = boto3.resource("s3").Bucket(context.TEST_ARTIFACT_BUCKET_NAME) + if bucket.creation_date is not None: + bucket.objects.all().delete() + bucket.delete() diff --git a/integration-tests/features/create-stack.feature b/integration-tests/features/create-stack.feature index 4dbd2b138..f482e2eeb 100644 --- a/integration-tests/features/create-stack.feature +++ b/integration-tests/features/create-stack.feature @@ -37,7 +37,7 @@ Feature: Create stack When the user creates stack "8/C" Then stack "8/C" exists in "ROLLBACK_COMPLETE" state - Scenario: create new stack that ignores dependencies + Scenario: create new stack that ignores dependencies Given stack "1/A" does not exist and the template for stack "1/A" is "valid_template.json" When the user creates stack "1/A" with ignore dependencies @@ -47,4 +47,15 @@ Feature: Create stack Given stack "10/A" does not exist and the template for stack "10/A" is "sam_template.yaml" When the user creates stack "10/A" - Then stack "10/A" exists in "CREATE_COMPLETE" state \ No newline at end of file + Then stack "10/A" exists in "CREATE_COMPLETE" state + + Scenario: create new stack with nested config jinja resolver + Given stack_group "12/1" does not exist + When the user launches stack_group "12/1" + Then all the stacks in stack_group "12/1" are in "CREATE_COMPLETE" + and stack "12/1/A" has "Project" tag with "A" value + and stack "12/1/A" has "Key" tag with "A" value + and stack "12/1/2/B" has "Project" tag with "B" value + and stack "12/1/2/B" has "Key" tag with "A-B" value + and stack "12/1/2/3/C" has "Project" tag with "C" value + and stack "12/1/2/3/C" has "Key" tag with "A-B-C" value diff --git a/integration-tests/features/delete-change-set.feature b/integration-tests/features/delete-change-set.feature index 67a0ffd0b..886dfd6d8 100644 --- a/integration-tests/features/delete-change-set.feature +++ b/integration-tests/features/delete-change-set.feature @@ -5,7 +5,7 @@ Feature: Delete change set and stack "1/A" has change set "A" using "updated_template.json" When the user deletes change set "A" for stack "1/A" Then stack "1/A" does not have change set "A" - + Scenario: delete a change set that exists with ignore dependencies Given stack "1/A" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/features/delete-stack.feature b/integration-tests/features/delete-stack.feature index 44deb555f..ff0f3cb9c 100644 --- a/integration-tests/features/delete-stack.feature +++ b/integration-tests/features/delete-stack.feature @@ -16,8 +16,26 @@ Feature: Delete stack Then stack "1/A" does not exist Scenario: delete a stack that exists with dependencies ignoring dependencies - Given stack "4/C" exists in "CREATE_COMPLETE" state - and stack "3/A" exists in "CREATE_COMPLETE" state + Given stack "4/C" exists in "CREATE_COMPLETE" state + and stack "3/A" exists in "CREATE_COMPLETE" state and stack "3/A" depends on stack "4/C" When the user deletes stack "4/C" with ignore dependencies Then stack "4/C" does not exist and stack "3/A" exists in "CREATE_COMPLETE" + + Scenario: delete a stack that contains !stack_output dependencies + Given stack "6/1/A" exists in "CREATE_COMPLETE" state + and stack "6/1/B" exists in "CREATE_COMPLETE" state + and stack "6/1/C" exists in "CREATE_COMPLETE" state + When the user deletes stack "6/1/A" + Then stack "6/1/A" does not exist + and stack "6/1/B" does not exist + and stack "6/1/C" does not exist + + Scenario: delete a stack that contains dependencies parameter + Given stack "3/A" exists in "CREATE_COMPLETE" state + and stack "3/B" exists in "CREATE_COMPLETE" state + and stack "3/C" exists in "CREATE_COMPLETE" state + When the user deletes stack "3/A" + Then stack "3/A" does not exist + and stack "3/B" does not exist + and stack "3/C" does not exist diff --git a/integration-tests/features/dependency-resolution.feature b/integration-tests/features/dependency-resolution.feature index a10a67e17..861e3e97d 100644 --- a/integration-tests/features/dependency-resolution.feature +++ b/integration-tests/features/dependency-resolution.feature @@ -2,17 +2,17 @@ Feature: Dependency resolution Scenario: launch a stack_group with dependencies that is partially complete Given stack "3/A" exists in "CREATE_COMPLETE" state - and stack "3/B" exists in "CREATE_COMPLETE" state - and stack "3/C" does not exist + And stack "3/B" exists in "CREATE_COMPLETE" state + And stack "3/C" does not exist When the user launches stack_group "3" Then all the stacks in stack_group "3" are in "CREATE_COMPLETE" - and that stack "3/A" was created before "3/B" - and that stack "3/B" was created before "3/C" + And that stack "3/A" was created before "3/B" + And that stack "3/B" was created before "3/C" Scenario: delete a stack_group with dependencies that is partially complete Given stack "3/A" exists in "CREATE_COMPLETE" state - and stack "3/B" exists in "CREATE_COMPLETE" state - and stack "3/C" does not exist + And stack "3/B" exists in "CREATE_COMPLETE" state + And stack "3/C" does not exist When the user deletes stack_group "3" Then all the stacks in stack_group "3" do not exist diff --git a/integration-tests/features/describe-change-set.feature b/integration-tests/features/describe-change-set.feature index 9cd99e0a5..d1fbfd6c7 100644 --- a/integration-tests/features/describe-change-set.feature +++ b/integration-tests/features/describe-change-set.feature @@ -18,5 +18,3 @@ Feature: Describe change sets and stack "1/A" has change set "A" using "updated_template.json" When the user describes change set "A" for stack "1/A" with ignore dependencies Then change set "A" for stack "1/A" is described - - diff --git a/integration-tests/features/describe-stack-group-resources.feature b/integration-tests/features/describe-stack-group-resources.feature index 6c3016065..1dd0f47f3 100644 --- a/integration-tests/features/describe-stack-group-resources.feature +++ b/integration-tests/features/describe-stack-group-resources.feature @@ -21,4 +21,3 @@ Feature: Describe stack_group resources Given all the stacks in stack_group "2" are in "CREATE_COMPLETE" When the user describes resources in stack_group "2" with ignore dependencies Then only all resources in stack_group "2" are described - diff --git a/integration-tests/features/describe-stack-group.feature b/integration-tests/features/describe-stack-group.feature index 9ac6b4012..8b1756de8 100644 --- a/integration-tests/features/describe-stack-group.feature +++ b/integration-tests/features/describe-stack-group.feature @@ -23,4 +23,3 @@ Feature: Describe stack_group Given all the stacks in stack_group "2" are in "CREATE_COMPLETE" When the user describes stack_group "2" with ignore dependencies Then all stacks in stack_group "2" are described as "CREATE_COMPLETE" - diff --git a/integration-tests/features/drift.feature b/integration-tests/features/drift.feature new file mode 100644 index 000000000..d628569d7 --- /dev/null +++ b/integration-tests/features/drift.feature @@ -0,0 +1,34 @@ +Feature: Drift Detection + Scenario: Detects no drift on a stack with no drift + Given stack "drift-single/A" exists using "topic.yaml" + When the user detects drift on stack "drift-single/A" + Then stack drift status is "IN_SYNC" + + Scenario: Shows no drift on a stack that with no drift + Given stack "drift-single/A" exists using "topic.yaml" + When the user shows drift on stack "drift-single/A" + Then stack resource drift status is "IN_SYNC" + + Scenario: Detects drift on a stack that has drifted + Given stack "drift-single/A" exists using "topic.yaml" + And a topic configuration in stack "drift-single/A" has drifted + When the user detects drift on stack "drift-single/A" + Then stack drift status is "DRIFTED" + + Scenario: Shows drift on a stack that has drifted + Given stack "drift-single/A" exists using "topic.yaml" + And a topic configuration in stack "drift-single/A" has drifted + When the user shows drift on stack "drift-single/A" + Then stack resource drift status is "MODIFIED" + + Scenario: Detects drift on a stack group that partially exists + Given stack "drift-group/A" exists using "topic.yaml" + And stack "drift-group/B" does not exist + And a topic configuration in stack "drift-group/A" has drifted + When the user detects drift on stack_group "drift-group" + Then stack_group drift statuses are each one of "DRIFTED,STACK_DOES_NOT_EXIST" + + Scenario: Does not blow up on a stack group that doesn't exist + Given stack_group "drift-group" does not exist + When the user detects drift on stack_group "drift-group" + Then stack_group drift statuses are each one of "STACK_DOES_NOT_EXIST,STACK_DOES_NOT_EXIST" diff --git a/integration-tests/features/execute-change-set.feature b/integration-tests/features/execute-change-set.feature index 68714e507..3ae3f650c 100644 --- a/integration-tests/features/execute-change-set.feature +++ b/integration-tests/features/execute-change-set.feature @@ -2,22 +2,33 @@ Feature: Execute change set Scenario: execute a change set that exists Given stack "1/A" exists in "CREATE_COMPLETE" state - and stack "1/A" has change set "A" using "updated_template.json" + And stack "1/A" has change set "A" using "updated_template.json" When the user executes change set "A" for stack "1/A" Then stack "1/A" does not have change set "A" - and stack "1/A" was updated with change set "A" + And stack "1/A" was updated with change set "A" Scenario: execute a change set that does not exist Given stack "1/A" exists in "CREATE_COMPLETE" state - and stack "1/A" does not have change set "A" + And stack "1/A" does not have change set "A" When the user executes change set "A" for stack "1/A" Then a "ClientError" is raised - and the user is told "change set does not exist" + And the user is told "change set does not exist" Scenario: execute a change set that exists with ignore dependencies Given stack "1/A" exists in "CREATE_COMPLETE" state - and stack "1/A" has change set "A" using "updated_template.json" + And stack "1/A" has change set "A" using "updated_template.json" When the user executes change set "A" for stack "1/A" with ignore dependencies Then stack "1/A" does not have change set "A" - and stack "1/A" was updated with change set "A" + And stack "1/A" was updated with change set "A" + Scenario: execute a change set that failed creation for no changes + Given stack "2/A" exists using "valid_template.json" + And stack "2/A" has change set "A" using "valid_template.json" + When the user executes change set "A" for stack "2/A" + Then stack "2/A" has change set "A" in "FAILED" state + + Scenario: execute a change set that failed creation for a SAM template with no changes + Given stack "3/A" exists using "sam_template.yaml" + And stack "3/A" has change set "A" using "sam_template.yaml" + When the user executes change set "A" for stack "3/A" + Then stack "3/A" has change set "A" in "FAILED" state diff --git a/integration-tests/features/generate-template-s3.feature b/integration-tests/features/generate-template-s3.feature new file mode 100644 index 000000000..e0ef65eb3 --- /dev/null +++ b/integration-tests/features/generate-template-s3.feature @@ -0,0 +1,17 @@ +@s3-template-handler +Feature: Generate template s3 + + Scenario: Generating static templates with S3 template handler + Given the template for stack "13/B" is "valid_template.json" + When the user generates the template for stack "13/B" + Then the output is the same as the contents of "valid_template.json" template + + Scenario: Render jinja templates with S3 template handler + Given the template for stack "13/C" is "jinja/valid_template.j2" + When the user generates the template for stack "13/C" + Then the output is the same as the contents of "valid_template.json" template + + Scenario: Render python templates with S3 template handler + Given the template for stack "13/D" is "python/valid_template.py" + When the user generates the template for stack "13/D" + Then the output is the same as the contents of "valid_template.json" template diff --git a/integration-tests/features/generate-template.feature b/integration-tests/features/generate-template.feature index b781a6059..97f374b8e 100644 --- a/integration-tests/features/generate-template.feature +++ b/integration-tests/features/generate-template.feature @@ -12,20 +12,26 @@ Feature: Generate template | invalid_template.json | | jinja/valid_template.json | | valid_template.yaml | + | valid_template_mark.yaml | | valid_template_func.yaml | | malformed_template.yaml | | invalid_template.yaml | | jinja/valid_template.yaml | - Scenario: Generate template using a valid python template file - Given the template for stack "1/A" is "valid_template.py" + Scenario: Generate template using a valid python template file that outputs json + Given the template for stack "1/A" is "valid_template_json.py" When the user generates the template for stack "1/A" - Then the output is the same as the string returned by "valid_template.py" + Then the output is the same as the contents returned by "valid_template_json.py" - Scenario: Generate template using a valid python template file with ignore dependencies - Given the template for stack "1/A" is "valid_template.py" + Scenario: Generate template using a valid python template file that outputs json with ignore dependencies + Given the template for stack "1/A" is "valid_template_json.py" When the user generates the template for stack "1/A" with ignore dependencies - Then the output is the same as the string returned by "valid_template.py" + Then the output is the same as the contents returned by "valid_template_json.py" + + Scenario: Generate template using a valid python template file that outputs yaml + Given the template for stack "1/A" is "valid_template_yaml.py" + When the user generates the template for stack "1/A" + Then the output is the same as the contents returned by "valid_template_yaml.py" Scenario Outline: Generating erroneous python templates Given the template for stack "1/A" is "" @@ -60,3 +66,8 @@ Feature: Generate template | filename | exception | | jinja/invalid_template_missing_key.j2 | UndefinedError | | jinja/invalid_template_missing_attr.j2 | UndefinedError | + + Scenario: Generating static templates with file template handler + Given the template for stack "13/A" is "valid_template.json" + When the user generates the template for stack "13/A" + Then the output is the same as the contents of "valid_template.json" template diff --git a/integration-tests/features/launch-stack-group.feature b/integration-tests/features/launch-stack-group.feature index 3602d2287..c78c6b2f1 100644 --- a/integration-tests/features/launch-stack-group.feature +++ b/integration-tests/features/launch-stack-group.feature @@ -4,7 +4,7 @@ Feature: Launch stack_group Given stack_group "2" does not exist When the user launches stack_group "2" Then all the stacks in stack_group "2" are in "CREATE_COMPLETE" - + Scenario: launch a stack_group, excluding dependencies, that does not exist Given stack_group "2" does not exist When the user launches stack_group "2" @@ -22,19 +22,19 @@ Feature: Launch stack_group Scenario: launch a stack_group with updates that partially exists Given stack "2/A" exists in "CREATE_COMPLETE" state - and stack "2/B" does not exist - and stack "2/C" does not exist - and the template for stack "2/A" is "updated_template.json" + And stack "2/B" does not exist + And stack "2/C" does not exist + And the template for stack "2/A" is "updated_template.json" When the user launches stack_group "2" Then stack "2/A" exists in "UPDATE_COMPLETE" state - and stack "2/B" exists in "CREATE_COMPLETE" state - and stack "2/C" exists in "CREATE_COMPLETE" state + And stack "2/B" exists in "CREATE_COMPLETE" state + And stack "2/C" exists in "CREATE_COMPLETE" state Scenario: launch a stack_group with updates that already exists Given all the stacks in stack_group "2" are in "CREATE_COMPLETE" - and the template for stack "2/A" is "updated_template.json" - and the template for stack "2/B" is "updated_template.json" - and the template for stack "2/C" is "updated_template.json" + And the template for stack "2/A" is "updated_template.json" + And the template for stack "2/B" is "updated_template.json" + And the template for stack "2/C" is "updated_template.json" When the user launches stack_group "2" Then all the stacks in stack_group "2" are in "UPDATE_COMPLETE" @@ -47,3 +47,31 @@ Feature: Launch stack_group Given stack_group "2" does not exist When the user launches stack_group "2" with ignore dependencies Then all the stacks in stack_group "2" are in "CREATE_COMPLETE" + + Scenario: launch a StackGroup with ignored and obsolete stacks that have not been launched + Given stack_group "launch-actions" does not exist + When the user launches stack_group "launch-actions" + Then stack "launch-actions/obsolete" does not exist + And stack "launch-actions/ignore" does not exist + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state + + Scenario: launch a StackGroup without --prune with obsolete stacks that currently exist + Given stack "launch-actions/obsolete" exists using "valid_template.json" + And stack "launch-actions/deploy" exists using "valid_template.json" + When the user launches stack_group "launch-actions" + Then stack "launch-actions/obsolete" exists in "CREATE_COMPLETE" state + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state + + Scenario: launch a StackGroup with --prune with obsolete stacks that currently exist + Given stack "launch-actions/obsolete" exists using "valid_template.json" + And stack "launch-actions/deploy" exists using "valid_template.json" + When the user launches stack_group "launch-actions" with --prune + Then stack "launch-actions/obsolete" does not exist + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state + + Scenario: launch a StackGroup with ignored stacks that currently exist + Given stack "launch-actions/ignore" exists using "valid_template.json" + And stack "launch-actions/deploy" exists using "valid_template.json" + When the user launches stack_group "launch-actions" + Then stack "launch-actions/ignore" exists in "CREATE_COMPLETE" state + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/features/launch-stack.feature b/integration-tests/features/launch-stack.feature index 39f58ddf8..73f41ab53 100644 --- a/integration-tests/features/launch-stack.feature +++ b/integration-tests/features/launch-stack.feature @@ -2,25 +2,49 @@ Feature: Launch stack Scenario: launch a new stack Given stack "1/A" does not exist - and the template for stack "1/A" is "valid_template.json" + And the template for stack "1/A" is "valid_template.json" When the user launches stack "1/A" Then stack "1/A" exists in "CREATE_COMPLETE" state Scenario: launch a stack that was newly created Given stack "1/A" exists in "CREATE_COMPLETE" state - and the template for stack "1/A" is "updated_template.json" + And the template for stack "1/A" is "updated_template.json" When the user launches stack "1/A" Then stack "1/A" exists in "UPDATE_COMPLETE" state Scenario: launch a stack that has been previously updated Given stack "1/A" exists in "UPDATE_COMPLETE" state - and the template for stack "1/A" is "valid_template.json" + And the template for stack "1/A" is "valid_template.json" When the user launches stack "1/A" Then stack "1/A" exists in "UPDATE_COMPLETE" state Scenario: launch a new stack with ignore dependencies Given stack "1/A" does not exist - and the template for stack "1/A" is "valid_template.json" + And the template for stack "1/A" is "valid_template.json" When the user launches stack "1/A" with ignore dependencies Then stack "1/A" exists in "CREATE_COMPLETE" state + Scenario: launch an obsolete stack that doesn't exist + Given stack "launch-actions/obsolete" does not exist + When the user launches stack "launch-actions/obsolete" + Then stack "launch-actions/obsolete" does not exist + + Scenario: launch an obsolete stack that does exist without --prune + Given stack "launch-actions/obsolete" exists using "valid_template.json" + When the user launches stack "launch-actions/obsolete" + Then stack "launch-actions/obsolete" exists in "CREATE_COMPLETE" state + + Scenario: launch an obsolete stack that does exist with --prune + Given stack "launch-actions/obsolete" exists using "valid_template.json" + When the user launches stack "launch-actions/obsolete" with --prune + Then stack "launch-actions/obsolete" does not exist + + Scenario: launch an ignored stack that doesn't exist + Given stack "launch-actions/ignore" does not exist + When the user launches stack "launch-actions/ignore" + Then stack "launch-actions/ignore" does not exist + + Scenario: launch an ignored stack that does exist + Given stack "launch-actions/ignore" exists using "valid_template.json" + When the user launches stack "launch-actions/ignore" + Then stack "launch-actions/ignore" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/features/list-change-sets.feature b/integration-tests/features/list-change-sets.feature index de8d53037..d2327f9d6 100644 --- a/integration-tests/features/list-change-sets.feature +++ b/integration-tests/features/list-change-sets.feature @@ -17,5 +17,3 @@ Feature: List change sets and stack "1/A" has change set "A" using "updated_template.json" When the user lists change sets for stack "1/A" with ignore dependencies Then the change sets for stack "1/A" are listed - - diff --git a/integration-tests/features/project-dependencies.feature b/integration-tests/features/project-dependencies.feature new file mode 100644 index 000000000..b7acfbb4d --- /dev/null +++ b/integration-tests/features/project-dependencies.feature @@ -0,0 +1,35 @@ +Feature: Project Dependencies managed within Sceptre + + Background: + Given stack_group "project-deps" does not exist + + Scenario: launch stack group with dependencies + Given all files in template bucket for stack "project-deps/main-project/resource" are deleted at cleanup + When the user launches stack_group "project-deps" + Then all the stacks in stack_group "project-deps" are in "CREATE_COMPLETE" + + Scenario: template_bucket_name is managed in stack group + Given all files in template bucket for stack "project-deps/main-project/resource" are deleted at cleanup + When the user launches stack_group "project-deps" + Then the template for stack "project-deps/main-project/resource" has been uploaded + + Scenario: notifications are managed in stack group + Given all files in template bucket for stack "project-deps/main-project/resource" are deleted at cleanup + When the user launches stack_group "project-deps" + Then the stack "project-deps/main-project/resource" has a notification defined by stack "project-deps/dependencies/topic" + + Scenario: validate a project that isn't deployed yet + Given placeholders are allowed + When the user validates stack_group "project-deps" + Then the user is told "the template is valid" + + Scenario: diff a project that isn't deployed yet + Given placeholders are allowed + When the user diffs stack group "project-deps" with "deepdiff" + Then a diff is returned with "is_deployed" = "False" + + Scenario: tags can be resolved + Given all files in template bucket for stack "project-deps/main-project/resource" are deleted at cleanup + When the user launches stack_group "project-deps" + Then the tag "greeting" for stack "project-deps/main-project/resource" is "hello" + And the tag "nonexistant" for stack "project-deps/main-project/resource" does not exist diff --git a/integration-tests/features/prune.feature b/integration-tests/features/prune.feature new file mode 100644 index 000000000..fd6baa402 --- /dev/null +++ b/integration-tests/features/prune.feature @@ -0,0 +1,22 @@ +Feature: Prune + + Scenario: Prune with no stacks marked obsolete does nothing + Given stack "pruning/not-obsolete" exists using "valid_template.json" + When command path "pruning/not-obsolete" is pruned + Then stack "pruning/not-obsolete" exists in "CREATE_COMPLETE" state + + Scenario: Prune whole project deletes all obsolete stacks that exist + Given all the stacks in stack_group "pruning" are in "CREATE_COMPLETE" + And stack "launch-actions/obsolete" exists using "valid_template.json" + When the whole project is pruned + Then stack "pruning/obsolete-1" does not exist + And stack "pruning/obsolete-2" does not exist + And stack "launch-actions/obsolete" does not exist + And stack "pruning/not-obsolete" exists in "CREATE_COMPLETE" state + + Scenario: Prune command path only deletes stacks on command path + Given stack "pruning/obsolete-1" exists using "valid_template.json" + And stack "pruning/obsolete-2" exists using "valid_template.json" + When command path "pruning/obsolete-1.yaml" is pruned + Then stack "pruning/obsolete-1" does not exist + And stack "pruning/obsolete-2" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/features/stack-diff.feature b/integration-tests/features/stack-diff.feature new file mode 100644 index 000000000..56f38aeb8 --- /dev/null +++ b/integration-tests/features/stack-diff.feature @@ -0,0 +1,80 @@ +Feature: Stack Diff + Scenario Outline: Diff on stack that exists with no changes + Given stack "1/A" exists in "CREATE_COMPLETE" state + And the template for stack "1/A" is "valid_template.json" + When the user diffs stack "1/A" with "" + Then a diff is returned with no "template" difference + And a diff is returned with no "config" difference + And a diff is returned with "is_deployed" = "True" + + Examples: DiffTypes + | diff_type | + | deepdiff | + | difflib | + + Scenario Outline: Diff on stack that doesnt exist + Given stack "1/A" does not exist + And the template for stack "1/A" is "valid_template.json" + When the user diffs stack "1/A" with "" + Then a diff is returned with "is_deployed" = "False" + And a diff is returned with a "template" difference + And a diff is returned with a "config" difference + + Examples: DiffTypes + | diff_type | + | deepdiff | + | difflib | + + Scenario Outline: Diff on stack that exists in non-deployed state + Given stack "1/A" exists in "" state + And the template for stack "1/A" is "valid_template.json" + When the user diffs stack "1/A" with "" + Then a diff is returned with a "template" difference + And a diff is returned with a "config" difference + And a diff is returned with "is_deployed" = "False" + + Examples: DeepDiff + | diff_type | status | + | deepdiff | CREATE_FAILED | + | deepdiff | ROLLBACK_COMPLETE | + | difflib | CREATE_FAILED | + | difflib | ROLLBACK_COMPLETE | + + Scenario Outline: Diff on stack with only template changes + Given stack "1/A" exists in "CREATE_COMPLETE" state + And the template for stack "1/A" is "updated_template.json" + When the user diffs stack "1/A" with "" + Then a diff is returned with a "template" difference + And a diff is returned with no "config" difference + + Examples: DiffTypes + | diff_type | + | deepdiff | + | difflib | + + Scenario Outline: Diff on stack with only configuration changes + Given stack "1/A" exists in "CREATE_COMPLETE" state + And the template for stack "1/A" is "valid_template.json" + And the stack config for stack "1/A" has changed + When the user diffs stack "1/A" with "" + Then a diff is returned with a "config" difference + And a diff is returned with no "template" difference + + Examples: DiffTypes + | diff_type | + | deepdiff | + | difflib | + + + Scenario Outline: Diff on stack with both configuration and template changes + Given stack "1/A" exists in "CREATE_COMPLETE" state + And the template for stack "1/A" is "updated_template.json" + And the stack config for stack "1/A" has changed + When the user diffs stack "1/A" with "" + Then a diff is returned with a "config" difference + And a diff is returned with a "template" difference + + Examples: DiffTypes + | diff_type | + | deepdiff | + | difflib | diff --git a/integration-tests/features/stack-output-external-resolver.feature b/integration-tests/features/stack-output-external-resolver.feature index 88321b930..c9ecbe07a 100644 --- a/integration-tests/features/stack-output-external-resolver.feature +++ b/integration-tests/features/stack-output-external-resolver.feature @@ -1,8 +1,13 @@ Feature: Stack output external resolver - Scenario: launch a stack referencing the external output of an existing stack - Given stack_group "9" has AWS config "aws-config" set - and stack "9/A" exists using "dependencies/independent_template.json" - and stack "9/B" does not exist - When the user launches stack "9/B" - Then stack "9/B" exists in "CREATE_COMPLETE" state + Scenario: launch a stack referencing the external output of an existing stack with region and profile + Given stack "external-stack-output/outputter" exists using "dependencies/independent_template.json" + And stack "external-stack-output/resolver-with-profile-region" does not exist + When the user launches stack "external-stack-output/resolver-with-profile-region" + Then stack "external-stack-output/resolver-with-profile-region" exists in "CREATE_COMPLETE" state + + Scenario: launch a stack referencing the external output of an existing stack without explicit region or profile + Given stack "external-stack-output-stack-output/outputter" exists using "dependencies/independent_template.json" + And stack "external-stack-output/resolver-no-profile-region" does not exist + When the user launches stack "external-stack-output/resolver-no-profile-region" + Then stack "external-stack-output/resolver-no-profile-region" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/features/update-stack.feature b/integration-tests/features/update-stack.feature index 1e01866fb..320895e1c 100644 --- a/integration-tests/features/update-stack.feature +++ b/integration-tests/features/update-stack.feature @@ -35,4 +35,4 @@ Feature: Update stack Given stack "11/A" exists in "CREATE_COMPLETE" state and the template for stack "11/A" is "sam_updated_template.yaml" When the user updates stack "11/A" - Then stack "11/A" exists in "UPDATE_COMPLETE" state \ No newline at end of file + Then stack "11/A" exists in "UPDATE_COMPLETE" state diff --git a/integration-tests/features/validate-template.feature b/integration-tests/features/validate-template.feature index 2c417207c..88b4a672b 100644 --- a/integration-tests/features/validate-template.feature +++ b/integration-tests/features/validate-template.feature @@ -11,8 +11,7 @@ Feature: Validate template Then a "ClientError" is raised and the user is told "the template is malformed" - Scenario: validate a vaild template with ignore dependencies + Scenario: validate a valid template with ignore dependencies Given the template for stack "1/A" is "valid_template.json" When the user validates the template for stack "1/A" with ignore dependencies Then the user is told "the template is valid" - diff --git a/integration-tests/sceptre-project/config/1/A.yaml b/integration-tests/sceptre-project/config/1/A.yaml index a35368f14..4bea28a07 100644 --- a/integration-tests/sceptre-project/config/1/A.yaml +++ b/integration-tests/sceptre-project/config/1/A.yaml @@ -1,2 +1,3 @@ stack_timeout: 1 -template_path: malformed_template.json +template: + path: malformed_template.json diff --git a/integration-tests/sceptre-project/config/10/A.yaml b/integration-tests/sceptre-project/config/10/A.yaml index 5842590f4..77f67b7c7 100644 --- a/integration-tests/sceptre-project/config/10/A.yaml +++ b/integration-tests/sceptre-project/config/10/A.yaml @@ -1 +1,2 @@ -template_path: sam_template.yaml +template: + path: sam_template.yaml diff --git a/integration-tests/sceptre-project/config/11/A.yaml b/integration-tests/sceptre-project/config/11/A.yaml index a7ec2dcea..6a97eac6c 100644 --- a/integration-tests/sceptre-project/config/11/A.yaml +++ b/integration-tests/sceptre-project/config/11/A.yaml @@ -1 +1,2 @@ -template_path: sam_updated_template.yaml +template: + path: sam_updated_template.yaml diff --git a/integration-tests/sceptre-project/config/12/1/2/3/C.yaml b/integration-tests/sceptre-project/config/12/1/2/3/C.yaml new file mode 100644 index 000000000..4c17329ad --- /dev/null +++ b/integration-tests/sceptre-project/config/12/1/2/3/C.yaml @@ -0,0 +1,5 @@ +template: + path: valid_template.json +stack_tags: + Project: '{{ project }}' + Key: '{{ keyC }}' diff --git a/integration-tests/sceptre-project/config/12/1/2/3/config.yaml b/integration-tests/sceptre-project/config/12/1/2/3/config.yaml new file mode 100644 index 000000000..5c18c3887 --- /dev/null +++ b/integration-tests/sceptre-project/config/12/1/2/3/config.yaml @@ -0,0 +1,2 @@ +keyC: "{{ keyB }}-C" +project: C diff --git a/integration-tests/sceptre-project/config/12/1/2/B.yaml b/integration-tests/sceptre-project/config/12/1/2/B.yaml new file mode 100644 index 000000000..1e9de974b --- /dev/null +++ b/integration-tests/sceptre-project/config/12/1/2/B.yaml @@ -0,0 +1,5 @@ +template: + path: valid_template.json +stack_tags: + Project: '{{ project }}' + Key: '{{ keyB }}' diff --git a/integration-tests/sceptre-project/config/12/1/2/config.yaml b/integration-tests/sceptre-project/config/12/1/2/config.yaml new file mode 100644 index 000000000..6d066d4f0 --- /dev/null +++ b/integration-tests/sceptre-project/config/12/1/2/config.yaml @@ -0,0 +1,2 @@ +keyB: '{{ keyA }}-B' +project: B diff --git a/integration-tests/sceptre-project/config/12/1/A.yaml b/integration-tests/sceptre-project/config/12/1/A.yaml new file mode 100644 index 000000000..dcb3a0c8c --- /dev/null +++ b/integration-tests/sceptre-project/config/12/1/A.yaml @@ -0,0 +1,5 @@ +template: + path: valid_template.json +stack_tags: + Key: '{{ keyA }}' + Project: '{{ project }}' diff --git a/integration-tests/sceptre-project/config/12/1/config.yaml b/integration-tests/sceptre-project/config/12/1/config.yaml new file mode 100644 index 000000000..fb70fdb42 --- /dev/null +++ b/integration-tests/sceptre-project/config/12/1/config.yaml @@ -0,0 +1,2 @@ +keyA: A +project: A diff --git a/integration-tests/sceptre-project/config/13/A.yaml b/integration-tests/sceptre-project/config/13/A.yaml new file mode 100644 index 000000000..1e97a39bb --- /dev/null +++ b/integration-tests/sceptre-project/config/13/A.yaml @@ -0,0 +1,3 @@ +template: + path: valid_template.json + type: file diff --git a/integration-tests/sceptre-project/config/13/B.yaml b/integration-tests/sceptre-project/config/13/B.yaml new file mode 100644 index 000000000..2396a313b --- /dev/null +++ b/integration-tests/sceptre-project/config/13/B.yaml @@ -0,0 +1,3 @@ +template: + path: sceptre-test-artifacts/13/valid_template.json + type: s3 diff --git a/integration-tests/sceptre-project/config/13/C.yaml b/integration-tests/sceptre-project/config/13/C.yaml new file mode 100644 index 000000000..5bb6fdb85 --- /dev/null +++ b/integration-tests/sceptre-project/config/13/C.yaml @@ -0,0 +1,5 @@ +sceptre_user_data: + type: AWS::CloudFormation::WaitConditionHandle +template: + path: sceptre-test-artifacts/13/jinja/valid_template.json + type: s3 diff --git a/integration-tests/sceptre-project/config/13/D.yaml b/integration-tests/sceptre-project/config/13/D.yaml new file mode 100644 index 000000000..1b988974d --- /dev/null +++ b/integration-tests/sceptre-project/config/13/D.yaml @@ -0,0 +1,5 @@ +sceptre_user_data: + type: AWS::CloudFormation::WaitConditionHandle +template: + path: sceptre-test-artifacts/13/python/valid_template.json + type: s3 diff --git a/integration-tests/sceptre-project/config/2/A.yaml b/integration-tests/sceptre-project/config/2/A.yaml index 87cf59424..b0e4f9065 100644 --- a/integration-tests/sceptre-project/config/2/A.yaml +++ b/integration-tests/sceptre-project/config/2/A.yaml @@ -1 +1,2 @@ -template_path: updated_template.json +template: + path: updated_template.json diff --git a/integration-tests/sceptre-project/config/2/B.yaml b/integration-tests/sceptre-project/config/2/B.yaml index 87cf59424..b0e4f9065 100644 --- a/integration-tests/sceptre-project/config/2/B.yaml +++ b/integration-tests/sceptre-project/config/2/B.yaml @@ -1 +1,2 @@ -template_path: updated_template.json +template: + path: updated_template.json diff --git a/integration-tests/sceptre-project/config/2/C.yaml b/integration-tests/sceptre-project/config/2/C.yaml index 87cf59424..b0e4f9065 100644 --- a/integration-tests/sceptre-project/config/2/C.yaml +++ b/integration-tests/sceptre-project/config/2/C.yaml @@ -1 +1,2 @@ -template_path: updated_template.json +template: + path: updated_template.json diff --git a/integration-tests/sceptre-project/config/3/A.yaml b/integration-tests/sceptre-project/config/3/A.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/3/A.yaml +++ b/integration-tests/sceptre-project/config/3/A.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/3/B.yaml b/integration-tests/sceptre-project/config/3/B.yaml index 514c5e3ea..ebe85fdbc 100644 --- a/integration-tests/sceptre-project/config/3/B.yaml +++ b/integration-tests/sceptre-project/config/3/B.yaml @@ -1,3 +1,4 @@ dependencies: - 3/A.yaml -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/3/C.yaml b/integration-tests/sceptre-project/config/3/C.yaml index f9e932a62..980d0690c 100644 --- a/integration-tests/sceptre-project/config/3/C.yaml +++ b/integration-tests/sceptre-project/config/3/C.yaml @@ -1,3 +1,4 @@ dependencies: - 3/B.yaml -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/4/A.yaml b/integration-tests/sceptre-project/config/4/A.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/4/A.yaml +++ b/integration-tests/sceptre-project/config/4/A.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/4/B.yaml b/integration-tests/sceptre-project/config/4/B.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/4/B.yaml +++ b/integration-tests/sceptre-project/config/4/B.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/4/C.yaml b/integration-tests/sceptre-project/config/4/C.yaml index d776e67d9..30842a6fe 100644 --- a/integration-tests/sceptre-project/config/4/C.yaml +++ b/integration-tests/sceptre-project/config/4/C.yaml @@ -1,3 +1,4 @@ -template_path: valid_template.json +template: + path: valid_template.json dependencies: - 3/A.yaml diff --git a/integration-tests/sceptre-project/config/5/1/A.yaml b/integration-tests/sceptre-project/config/5/1/A.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/5/1/A.yaml +++ b/integration-tests/sceptre-project/config/5/1/A.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/5/1/B.yaml b/integration-tests/sceptre-project/config/5/1/B.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/5/1/B.yaml +++ b/integration-tests/sceptre-project/config/5/1/B.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/5/1/C.yaml b/integration-tests/sceptre-project/config/5/1/C.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/5/1/C.yaml +++ b/integration-tests/sceptre-project/config/5/1/C.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/5/2/A.yaml b/integration-tests/sceptre-project/config/5/2/A.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/5/2/A.yaml +++ b/integration-tests/sceptre-project/config/5/2/A.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/5/2/B.yaml b/integration-tests/sceptre-project/config/5/2/B.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/5/2/B.yaml +++ b/integration-tests/sceptre-project/config/5/2/B.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/5/2/C.yaml b/integration-tests/sceptre-project/config/5/2/C.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/5/2/C.yaml +++ b/integration-tests/sceptre-project/config/5/2/C.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/6/1/A.yaml b/integration-tests/sceptre-project/config/6/1/A.yaml index 70c25a81c..42dd6738a 100644 --- a/integration-tests/sceptre-project/config/6/1/A.yaml +++ b/integration-tests/sceptre-project/config/6/1/A.yaml @@ -1 +1,2 @@ -template_path: dependencies/independent_template.json +template: + path: dependencies/independent_template.json diff --git a/integration-tests/sceptre-project/config/6/1/B.yaml b/integration-tests/sceptre-project/config/6/1/B.yaml index fbd458ec8..4bf13facf 100644 --- a/integration-tests/sceptre-project/config/6/1/B.yaml +++ b/integration-tests/sceptre-project/config/6/1/B.yaml @@ -1,3 +1,4 @@ -template_path: dependencies/dependent_template.json +template: + path: dependencies/dependent_template.json parameters: DependentStackName: !stack_output 6/1/A.yaml::StackName diff --git a/integration-tests/sceptre-project/config/6/1/C.yaml b/integration-tests/sceptre-project/config/6/1/C.yaml index c2c5560e9..533e16a94 100644 --- a/integration-tests/sceptre-project/config/6/1/C.yaml +++ b/integration-tests/sceptre-project/config/6/1/C.yaml @@ -1,3 +1,4 @@ -template_path: dependencies/dependent_template.json +template: + path: dependencies/dependent_template.json parameters: DependentStackName: !stack_output 6/1/B.yaml::StackName diff --git a/integration-tests/sceptre-project/config/6/2/A.yaml b/integration-tests/sceptre-project/config/6/2/A.yaml index 42549a093..7832e2386 100644 --- a/integration-tests/sceptre-project/config/6/2/A.yaml +++ b/integration-tests/sceptre-project/config/6/2/A.yaml @@ -1,2 +1,3 @@ -template_path: dependencies/independent_template.json +template: + path: dependencies/independent_template.json region: eu-west-1 diff --git a/integration-tests/sceptre-project/config/6/2/B.yaml b/integration-tests/sceptre-project/config/6/2/B.yaml index 03a4ce3c0..a0c604bc6 100644 --- a/integration-tests/sceptre-project/config/6/2/B.yaml +++ b/integration-tests/sceptre-project/config/6/2/B.yaml @@ -1,4 +1,5 @@ -template_path: dependencies/dependent_template_local_export.json +template: + path: dependencies/dependent_template_local_export.json region: eu-west-2 parameters: DependentStackName: !stack_output 6/2/A.yaml::StackName diff --git a/integration-tests/sceptre-project/config/6/2/C.yaml b/integration-tests/sceptre-project/config/6/2/C.yaml index d898ebd51..712f8f1c3 100644 --- a/integration-tests/sceptre-project/config/6/2/C.yaml +++ b/integration-tests/sceptre-project/config/6/2/C.yaml @@ -1,4 +1,5 @@ -template_path: dependencies/dependent_template_local_export.json +template: + path: dependencies/dependent_template_local_export.json region: eu-west-3 parameters: DependentStackName: !stack_output 6/2/B.yaml::StackName diff --git a/integration-tests/sceptre-project/config/6/3/A.yaml b/integration-tests/sceptre-project/config/6/3/A.yaml index fbb2444bf..1cf4462fc 100644 --- a/integration-tests/sceptre-project/config/6/3/A.yaml +++ b/integration-tests/sceptre-project/config/6/3/A.yaml @@ -1,3 +1,4 @@ -template_path: output_template.json +template: + path: output_template.json parameters: Input: "TestValue" diff --git a/integration-tests/sceptre-project/config/6/3/B.yaml b/integration-tests/sceptre-project/config/6/3/B.yaml index 18a4193ce..1f2bb13a9 100644 --- a/integration-tests/sceptre-project/config/6/3/B.yaml +++ b/integration-tests/sceptre-project/config/6/3/B.yaml @@ -1,3 +1,4 @@ -template_path: input_output_template.json +template: + path: input_output_template.json parameters: Input: !stack_output 6/3/A.yaml::Output diff --git a/integration-tests/sceptre-project/config/6/3/C.yaml b/integration-tests/sceptre-project/config/6/3/C.yaml index 615cb3311..00ff1711a 100644 --- a/integration-tests/sceptre-project/config/6/3/C.yaml +++ b/integration-tests/sceptre-project/config/6/3/C.yaml @@ -1,3 +1,4 @@ -template_path: input_output_template.json +template: + path: input_output_template.json parameters: Input: !stack_output 6/3/B.yaml::Output diff --git a/integration-tests/sceptre-project/config/6/4/1/A.yaml b/integration-tests/sceptre-project/config/6/4/1/A.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/6/4/1/A.yaml +++ b/integration-tests/sceptre-project/config/6/4/1/A.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/6/4/1/B.yaml b/integration-tests/sceptre-project/config/6/4/1/B.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/6/4/1/B.yaml +++ b/integration-tests/sceptre-project/config/6/4/1/B.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/6/4/1/C.yaml b/integration-tests/sceptre-project/config/6/4/1/C.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/6/4/1/C.yaml +++ b/integration-tests/sceptre-project/config/6/4/1/C.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/6/4/2/A.yaml b/integration-tests/sceptre-project/config/6/4/2/A.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/6/4/2/A.yaml +++ b/integration-tests/sceptre-project/config/6/4/2/A.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/6/4/2/B.yaml b/integration-tests/sceptre-project/config/6/4/2/B.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/6/4/2/B.yaml +++ b/integration-tests/sceptre-project/config/6/4/2/B.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/6/4/2/C.yaml b/integration-tests/sceptre-project/config/6/4/2/C.yaml index c74618e06..0d905dfe0 100644 --- a/integration-tests/sceptre-project/config/6/4/2/C.yaml +++ b/integration-tests/sceptre-project/config/6/4/2/C.yaml @@ -1 +1,2 @@ -template_path: valid_template.json +template: + path: valid_template.json diff --git a/integration-tests/sceptre-project/config/7/A.yaml b/integration-tests/sceptre-project/config/7/A.yaml index cb292d02f..c94ce5bb6 100644 --- a/integration-tests/sceptre-project/config/7/A.yaml +++ b/integration-tests/sceptre-project/config/7/A.yaml @@ -1,3 +1,4 @@ sceptre_user_data: type: AWS::CloudFormation::WaitConditionHandle -template_path: invalid_template_missing_attr.j2 +template: + path: invalid_template_missing_attr.j2 diff --git a/integration-tests/sceptre-project/config/8/A.yaml b/integration-tests/sceptre-project/config/8/A.yaml index ba1f80440..ae471013d 100644 --- a/integration-tests/sceptre-project/config/8/A.yaml +++ b/integration-tests/sceptre-project/config/8/A.yaml @@ -1 +1,2 @@ -template_path: invalid_template.json +template: + path: invalid_template.json diff --git a/integration-tests/sceptre-project/config/8/B.yaml b/integration-tests/sceptre-project/config/8/B.yaml index a804a2ff3..42fee7e6c 100644 --- a/integration-tests/sceptre-project/config/8/B.yaml +++ b/integration-tests/sceptre-project/config/8/B.yaml @@ -1,2 +1,3 @@ on_failure: DO_NOTHING -template_path: invalid_template.json +template: + path: invalid_template.json diff --git a/integration-tests/sceptre-project/config/8/C.yaml b/integration-tests/sceptre-project/config/8/C.yaml index 6728bff2f..15ee72a33 100644 --- a/integration-tests/sceptre-project/config/8/C.yaml +++ b/integration-tests/sceptre-project/config/8/C.yaml @@ -1,2 +1,3 @@ stack_timeout: 1 -template_path: valid_template_wait_300.json +template: + path: valid_template_wait_300.json diff --git a/integration-tests/sceptre-project/config/9/A.yaml b/integration-tests/sceptre-project/config/9/A.yaml deleted file mode 100644 index 42549a093..000000000 --- a/integration-tests/sceptre-project/config/9/A.yaml +++ /dev/null @@ -1,2 +0,0 @@ -template_path: dependencies/independent_template.json -region: eu-west-1 diff --git a/integration-tests/sceptre-project/config/9/B.yaml b/integration-tests/sceptre-project/config/9/B.yaml deleted file mode 100644 index fc85b4425..000000000 --- a/integration-tests/sceptre-project/config/9/B.yaml +++ /dev/null @@ -1,4 +0,0 @@ -template_path: dependencies/dependent_template_local_export.json -region: eu-west-1 -parameters: - DependentStackName: !stack_output_external {project_code}-9-A::StackName default::eu-west-1 diff --git a/integration-tests/sceptre-project/config/9/aws-config b/integration-tests/sceptre-project/config/9/aws-config deleted file mode 100644 index a8c11c6e8..000000000 --- a/integration-tests/sceptre-project/config/9/aws-config +++ /dev/null @@ -1,2 +0,0 @@ -[default] -region = us-east-1 diff --git a/integration-tests/sceptre-project/config/drift-group/A.yaml b/integration-tests/sceptre-project/config/drift-group/A.yaml new file mode 100644 index 000000000..07215b35c --- /dev/null +++ b/integration-tests/sceptre-project/config/drift-group/A.yaml @@ -0,0 +1,3 @@ +template: + path: loggroup.yaml + type: file diff --git a/integration-tests/sceptre-project/config/drift-group/B.yaml b/integration-tests/sceptre-project/config/drift-group/B.yaml new file mode 100644 index 000000000..07215b35c --- /dev/null +++ b/integration-tests/sceptre-project/config/drift-group/B.yaml @@ -0,0 +1,3 @@ +template: + path: loggroup.yaml + type: file diff --git a/integration-tests/sceptre-project/config/drift-single/A.yaml b/integration-tests/sceptre-project/config/drift-single/A.yaml new file mode 100644 index 000000000..07215b35c --- /dev/null +++ b/integration-tests/sceptre-project/config/drift-single/A.yaml @@ -0,0 +1,3 @@ +template: + path: loggroup.yaml + type: file diff --git a/integration-tests/sceptre-project/config/external-stack-output/outputter.yaml b/integration-tests/sceptre-project/config/external-stack-output/outputter.yaml new file mode 100644 index 000000000..7832e2386 --- /dev/null +++ b/integration-tests/sceptre-project/config/external-stack-output/outputter.yaml @@ -0,0 +1,3 @@ +template: + path: dependencies/independent_template.json +region: eu-west-1 diff --git a/integration-tests/sceptre-project/config/external-stack-output/resolver-no-profile-region.yaml b/integration-tests/sceptre-project/config/external-stack-output/resolver-no-profile-region.yaml new file mode 100644 index 000000000..5b2f6b9a1 --- /dev/null +++ b/integration-tests/sceptre-project/config/external-stack-output/resolver-no-profile-region.yaml @@ -0,0 +1,5 @@ +template: + path: dependencies/dependent_template_local_export.json + +parameters: + DependentStackName: !stack_output_external "{{project_code}}-external-stack-output-outputter::StackName" diff --git a/integration-tests/sceptre-project/config/external-stack-output/resolver-with-profile-region.yaml b/integration-tests/sceptre-project/config/external-stack-output/resolver-with-profile-region.yaml new file mode 100644 index 000000000..b40ad7e42 --- /dev/null +++ b/integration-tests/sceptre-project/config/external-stack-output/resolver-with-profile-region.yaml @@ -0,0 +1,6 @@ +template: + path: dependencies/dependent_template_local_export.json + + +parameters: + DependentStackName: !stack_output_external "{{project_code}}-external-stack-output-outputter::StackName {{environment_variable.AWS_PROFILE|default('default')}}::eu-west-1" diff --git a/integration-tests/sceptre-project/config/launch-actions/deploy.yaml b/integration-tests/sceptre-project/config/launch-actions/deploy.yaml new file mode 100644 index 000000000..4d3e36dc5 --- /dev/null +++ b/integration-tests/sceptre-project/config/launch-actions/deploy.yaml @@ -0,0 +1,2 @@ +template: + path: valid_template.yaml diff --git a/integration-tests/sceptre-project/config/launch-actions/ignore.yaml b/integration-tests/sceptre-project/config/launch-actions/ignore.yaml new file mode 100644 index 000000000..4123a82ac --- /dev/null +++ b/integration-tests/sceptre-project/config/launch-actions/ignore.yaml @@ -0,0 +1,4 @@ +ignore: True + +template: + path: valid_template.yaml diff --git a/integration-tests/sceptre-project/config/launch-actions/obsolete.yaml b/integration-tests/sceptre-project/config/launch-actions/obsolete.yaml new file mode 100644 index 000000000..afe30525f --- /dev/null +++ b/integration-tests/sceptre-project/config/launch-actions/obsolete.yaml @@ -0,0 +1,4 @@ +obsolete: True + +template: + path: valid_template.yaml diff --git a/integration-tests/sceptre-project/config/project-deps/dependencies/assumed-role.yaml b/integration-tests/sceptre-project/config/project-deps/dependencies/assumed-role.yaml new file mode 100644 index 000000000..f8e70ec84 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/dependencies/assumed-role.yaml @@ -0,0 +1,2 @@ +template: + path: "project-dependencies/assumed-role.yaml" diff --git a/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml b/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml new file mode 100644 index 000000000..8a67bf664 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml @@ -0,0 +1,2 @@ +template: + path: project-dependencies/bucket.yaml diff --git a/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml b/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml new file mode 100644 index 000000000..8593392f3 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml @@ -0,0 +1,2 @@ +template: + path: project-dependencies/topic.yaml diff --git a/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml b/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml new file mode 100644 index 000000000..f03ad5867 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml @@ -0,0 +1,9 @@ +template_bucket_name: !stack_output project-deps/dependencies/bucket.yaml::BucketName +notifications: + - !stack_output project-deps/dependencies/topic.yaml::TopicArn + +sceptre_role: !stack_output project-deps/dependencies/assumed-role.yaml::RoleArn +sceptre_role_session_duration: 1800 +stack_tags: + greeting: !rcmd "echo 'hello' | tr -d '\n'" + nonexistant: !no_value diff --git a/integration-tests/sceptre-project/config/project-deps/main-project/resource.yaml b/integration-tests/sceptre-project/config/project-deps/main-project/resource.yaml new file mode 100644 index 000000000..339227ae6 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/main-project/resource.yaml @@ -0,0 +1,2 @@ +template: + path: "valid_template.yaml" diff --git a/integration-tests/sceptre-project/config/pruning/not-obsolete.yaml b/integration-tests/sceptre-project/config/pruning/not-obsolete.yaml new file mode 100644 index 000000000..b195ba2a7 --- /dev/null +++ b/integration-tests/sceptre-project/config/pruning/not-obsolete.yaml @@ -0,0 +1,3 @@ +template: + path: /Users/jfalkenstein/sceptre/integration-tests/sceptre-project/templates/valid_template.json + type: file diff --git a/integration-tests/sceptre-project/config/pruning/obsolete-1.yaml b/integration-tests/sceptre-project/config/pruning/obsolete-1.yaml new file mode 100644 index 000000000..ba377c5ea --- /dev/null +++ b/integration-tests/sceptre-project/config/pruning/obsolete-1.yaml @@ -0,0 +1,4 @@ +obsolete: true +template: + path: /Users/jfalkenstein/sceptre/integration-tests/sceptre-project/templates/valid_template.json + type: file diff --git a/integration-tests/sceptre-project/config/pruning/obsolete-2.yaml b/integration-tests/sceptre-project/config/pruning/obsolete-2.yaml new file mode 100644 index 000000000..ba377c5ea --- /dev/null +++ b/integration-tests/sceptre-project/config/pruning/obsolete-2.yaml @@ -0,0 +1,4 @@ +obsolete: true +template: + path: /Users/jfalkenstein/sceptre/integration-tests/sceptre-project/templates/valid_template.json + type: file diff --git a/integration-tests/sceptre-project/templates/invalid_template.yaml b/integration-tests/sceptre-project/templates/invalid_template.yaml index 9f07b118b..25180e3fb 100644 --- a/integration-tests/sceptre-project/templates/invalid_template.yaml +++ b/integration-tests/sceptre-project/templates/invalid_template.yaml @@ -1,5 +1,5 @@ Resources: - WaitConditionHandle: - Type: "AWS::CloudFormation::WaitConditionHandle" - Properties: - Invalid: Invalid + WaitConditionHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + Properties: + Invalid: Invalid diff --git a/integration-tests/sceptre-project/templates/jinja/valid_template.j2 b/integration-tests/sceptre-project/templates/jinja/valid_template.j2 index bb7f35bb4..09d89d2c0 100644 --- a/integration-tests/sceptre-project/templates/jinja/valid_template.j2 +++ b/integration-tests/sceptre-project/templates/jinja/valid_template.j2 @@ -1,4 +1,4 @@ Resources: WaitConditionHandle: Type: "{{ sceptre_user_data.type }}" - Properties: + Properties: {} diff --git a/integration-tests/sceptre-project/templates/jinja/valid_template.yaml b/integration-tests/sceptre-project/templates/jinja/valid_template.yaml index bb7f35bb4..e75d6e2d3 100644 --- a/integration-tests/sceptre-project/templates/jinja/valid_template.yaml +++ b/integration-tests/sceptre-project/templates/jinja/valid_template.yaml @@ -1,4 +1,4 @@ Resources: - WaitConditionHandle: - Type: "{{ sceptre_user_data.type }}" - Properties: + WaitConditionHandle: + Type: "{{ sceptre_user_data.type }}" + Properties: diff --git a/integration-tests/sceptre-project/templates/malformed_template.yaml b/integration-tests/sceptre-project/templates/malformed_template.yaml index c7aba0cb4..751319c53 100644 --- a/integration-tests/sceptre-project/templates/malformed_template.yaml +++ b/integration-tests/sceptre-project/templates/malformed_template.yaml @@ -1,4 +1,4 @@ Malformed: - WaitConditionHandle: - Type: "AWS::CloudFormation::WaitConditionHandle" - Malformed: + WaitConditionHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + Malformed: diff --git a/integration-tests/sceptre-project/templates/missing_sceptre_handler.py b/integration-tests/sceptre-project/templates/missing_sceptre_handler.py index 54b08a6ad..244a421fe 100644 --- a/integration-tests/sceptre-project/templates/missing_sceptre_handler.py +++ b/integration-tests/sceptre-project/templates/missing_sceptre_handler.py @@ -1,2 +1,2 @@ -if __name__ == '__main__': +if __name__ == "__main__": pass diff --git a/integration-tests/sceptre-project/templates/project-dependencies/assumed-role.yaml b/integration-tests/sceptre-project/templates/project-dependencies/assumed-role.yaml new file mode 100644 index 000000000..43f0c0f77 --- /dev/null +++ b/integration-tests/sceptre-project/templates/project-dependencies/assumed-role.yaml @@ -0,0 +1,31 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Resources: + Role: + Type: AWS::IAM::Role + Properties: + Path: /service/ + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: !Ref AWS::AccountId + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AWSCloudFormationFullAccess + - arn:aws:iam::aws:policy/AmazonS3FullAccess + - arn:aws:iam::aws:policy/AmazonSNSFullAccess + Policies: + - PolicyName: "PassRolePermissions" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Action: "iam:PassRole" + Effect: Allow + Resource: "*" + MaxSessionDuration: 43200 + +Outputs: + RoleArn: + Value: !GetAtt Role.Arn diff --git a/integration-tests/sceptre-project/templates/project-dependencies/bucket.yaml b/integration-tests/sceptre-project/templates/project-dependencies/bucket.yaml new file mode 100644 index 000000000..69b5c76e3 --- /dev/null +++ b/integration-tests/sceptre-project/templates/project-dependencies/bucket.yaml @@ -0,0 +1,10 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: { } + +Outputs: + BucketName: + Value: !Ref Bucket diff --git a/integration-tests/sceptre-project/templates/project-dependencies/topic.yaml b/integration-tests/sceptre-project/templates/project-dependencies/topic.yaml new file mode 100644 index 000000000..806145ea9 --- /dev/null +++ b/integration-tests/sceptre-project/templates/project-dependencies/topic.yaml @@ -0,0 +1,10 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Resources: + Topic: + Type: AWS::SNS::Topic + Properties: {} + +Outputs: + TopicArn: + Value: !Ref Topic diff --git a/integration-tests/sceptre-project/templates/python/valid_template.py b/integration-tests/sceptre-project/templates/python/valid_template.py new file mode 100644 index 000000000..dafffbc27 --- /dev/null +++ b/integration-tests/sceptre-project/templates/python/valid_template.py @@ -0,0 +1,8 @@ +from troposphere import Template +from troposphere.cloudformation import WaitConditionHandle + + +def sceptre_handler(scepter_user_data): + template = Template() + template.add_resource(WaitConditionHandle("WaitConditionHandle")) + return template.to_json() diff --git a/integration-tests/sceptre-project/templates/sam_template.yaml b/integration-tests/sceptre-project/templates/sam_template.yaml index 713361359..2985d99a4 100644 --- a/integration-tests/sceptre-project/templates/sam_template.yaml +++ b/integration-tests/sceptre-project/templates/sam_template.yaml @@ -5,5 +5,5 @@ Resources: Type: 'AWS::Serverless::Function' Properties: Handler: index.handler - Runtime: python3.6 + Runtime: python3.9 InlineCode: "print('Hello World!')" diff --git a/integration-tests/sceptre-project/templates/sam_updated_template.yaml b/integration-tests/sceptre-project/templates/sam_updated_template.yaml index 537d4deb7..c0630953c 100644 --- a/integration-tests/sceptre-project/templates/sam_updated_template.yaml +++ b/integration-tests/sceptre-project/templates/sam_updated_template.yaml @@ -1,9 +1,9 @@ Transform: 'AWS::Serverless-2016-10-31' Resources: - TestFunction2: - Type: 'AWS::Serverless::Function' - Properties: - Handler: index.handler - Runtime: python3.6 - InlineCode: "print('Hello Again World!')" \ No newline at end of file + TestFunction2: + Type: 'AWS::Serverless::Function' + Properties: + Handler: index.handler + Runtime: python3.9 + InlineCode: "print('Hello Again World!')" diff --git a/integration-tests/sceptre-project/templates/topic.yaml b/integration-tests/sceptre-project/templates/topic.yaml new file mode 100644 index 000000000..ab114cc48 --- /dev/null +++ b/integration-tests/sceptre-project/templates/topic.yaml @@ -0,0 +1,11 @@ +Resources: + Topic: + Type: AWS::SNS::Topic + Properties: + DisplayName: MyTopic + +Outputs: + TopicName: + Value: !Ref Topic + Export: + Name: !Sub "${AWS::StackName}-TopicName" diff --git a/integration-tests/sceptre-project/templates/valid_template.py b/integration-tests/sceptre-project/templates/valid_template.py deleted file mode 100644 index 2bbeb2a27..000000000 --- a/integration-tests/sceptre-project/templates/valid_template.py +++ /dev/null @@ -1,13 +0,0 @@ -import json - - -def sceptre_handler(scepter_user_data): - template = { - "Resources": { - "WaitConditionHandle": { - "Type": "AWS::CloudFormation::WaitConditionHandle", - "Properties": {} - } - } - } - return json.dumps(template) diff --git a/integration-tests/sceptre-project/templates/valid_template.yaml b/integration-tests/sceptre-project/templates/valid_template.yaml index c31e3ad34..9843e4fb2 100644 --- a/integration-tests/sceptre-project/templates/valid_template.yaml +++ b/integration-tests/sceptre-project/templates/valid_template.yaml @@ -1,4 +1,4 @@ Resources: - WaitConditionHandle: - Type: "AWS::CloudFormation::WaitConditionHandle" - Properties: + WaitConditionHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + Properties: {} diff --git a/integration-tests/sceptre-project/templates/valid_template_func.yaml b/integration-tests/sceptre-project/templates/valid_template_func.yaml index 879329c7d..d8592dc60 100644 --- a/integration-tests/sceptre-project/templates/valid_template_func.yaml +++ b/integration-tests/sceptre-project/templates/valid_template_func.yaml @@ -1,8 +1,8 @@ Resources: - WaitConditionHandle: - Type: "AWS:CloudFormation::WaitConditionHandle" - WaitCondition: - Type: "AWS::CloudFormation::WaitCondition" - Properties: - Count: 1 - Handle: !Ref WaitConditionHandle + WaitConditionHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + WaitCondition: + Type: "AWS::CloudFormation::WaitCondition" + Properties: + Count: 1 + Handle: !Ref WaitConditionHandle diff --git a/integration-tests/sceptre-project/templates/valid_template_json.py b/integration-tests/sceptre-project/templates/valid_template_json.py new file mode 100644 index 000000000..ef7813a79 --- /dev/null +++ b/integration-tests/sceptre-project/templates/valid_template_json.py @@ -0,0 +1,13 @@ +import json + + +def sceptre_handler(scepter_user_data): + template = { + "Resources": { + "WaitConditionHandle": { + "Type": "AWS::CloudFormation::WaitConditionHandle", + "Properties": {}, + } + } + } + return json.dumps(template) diff --git a/integration-tests/sceptre-project/templates/valid_template_mark.yaml b/integration-tests/sceptre-project/templates/valid_template_mark.yaml new file mode 100644 index 000000000..bdc07ce1b --- /dev/null +++ b/integration-tests/sceptre-project/templates/valid_template_mark.yaml @@ -0,0 +1,5 @@ +--- +Resources: + WaitConditionHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + Properties: diff --git a/integration-tests/sceptre-project/templates/valid_template_yaml.py b/integration-tests/sceptre-project/templates/valid_template_yaml.py new file mode 100644 index 000000000..e7c209ea1 --- /dev/null +++ b/integration-tests/sceptre-project/templates/valid_template_yaml.py @@ -0,0 +1,13 @@ +import yaml + + +def sceptre_handler(scepter_user_data): + template = { + "Resources": { + "WaitConditionHandle": { + "Type": "AWS::CloudFormation::WaitConditionHandle", + "Properties": {}, + } + } + } + return yaml.dump(template) diff --git a/integration-tests/steps/change_sets.py b/integration-tests/steps/change_sets.py index 4f140ec1c..2c5fa0c07 100644 --- a/integration-tests/steps/change_sets.py +++ b/integration-tests/steps/change_sets.py @@ -8,17 +8,19 @@ from helpers import retry_boto_call -@given( - 'stack "{stack_name}" has change set "{change_set_name}" using "{filename}"' -) +@given('stack "{stack_name}" has change set "{change_set_name}" using "{filename}"') def step_impl(context, stack_name, change_set_name, filename): full_name = get_cloudformation_stack_name(context, stack_name) retry_boto_call( context.client.create_change_set, StackName=full_name, - Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND'], + Capabilities=[ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], ChangeSetName=change_set_name, - TemplateBody=read_template_file(context, filename) + TemplateBody=read_template_file(context, filename), ) wait_for_final_state(context, stack_name, change_set_name) @@ -29,38 +31,35 @@ def step_impl(context, stack_name, change_set_name): retry_boto_call( context.client.delete_change_set, ChangeSetName=change_set_name, - StackName=full_name + StackName=full_name, ) @given('stack "{stack_name}" has no change sets') def step_impl(context, stack_name): full_name = get_cloudformation_stack_name(context, stack_name) - response = retry_boto_call( - context.client.list_change_sets, StackName=full_name - ) + response = retry_boto_call(context.client.list_change_sets, StackName=full_name) for change_set in response["Summaries"]: time.sleep(1) retry_boto_call( context.client.delete_change_set, - ChangeSetName=change_set['ChangeSetName'], - StackName=full_name + ChangeSetName=change_set["ChangeSetName"], + StackName=full_name, ) @when('the user creates change set "{change_set_name}" for stack "{stack_name}"') def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: sceptre_plan.create_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: @@ -69,20 +68,22 @@ def step_impl(context, change_set_name, stack_name): wait_for_final_state(context, stack_name, change_set_name) -@when('the user creates change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies') +@when( + 'the user creates change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies' +) def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: sceptre_plan.create_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: @@ -94,40 +95,41 @@ def step_impl(context, change_set_name, stack_name): @when('the user deletes change set "{change_set_name}" for stack "{stack_name}"') def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} sceptre_plan.delete_change_set(change_set_name) try: sceptre_plan.delete_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: raise e -@when('the user deletes change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies') +@when( + 'the user deletes change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies' +) def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} sceptre_plan.delete_change_set(change_set_name) try: sceptre_plan.delete_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: @@ -137,17 +139,16 @@ def step_impl(context, change_set_name, stack_name): @when('the user lists change sets for stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: context.output = sceptre_plan.list_change_sets().values() except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return raise e @@ -156,18 +157,18 @@ def step_impl(context, stack_name): @when('the user lists change sets for stack "{stack_name}" with ignore dependencies') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: context.output = sceptre_plan.list_change_sets().values() except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return raise e @@ -176,38 +177,39 @@ def step_impl(context, stack_name): @when('the user executes change set "{change_set_name}" for stack "{stack_name}"') def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: sceptre_plan.execute_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: raise e -@when('the user executes change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies') +@when( + 'the user executes change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies' +) def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: sceptre_plan.execute_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: @@ -217,17 +219,16 @@ def step_impl(context, change_set_name, stack_name): @when('the user describes change set "{change_set_name}" for stack "{stack_name}"') def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: responses = sceptre_plan.describe_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: @@ -235,21 +236,23 @@ def step_impl(context, change_set_name, stack_name): context.output = responses -@when('the user describes change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies') +@when( + 'the user describes change set "{change_set_name}" for stack "{stack_name}" with ignore dependencies' +) def step_impl(context, change_set_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) - allowed_errors = {'ValidationError', 'ChangeSetNotFound'} + allowed_errors = {"ValidationError", "ChangeSetNotFound"} try: responses = sceptre_plan.describe_change_set(change_set_name) except ClientError as e: - if e.response['Error']['Code'] in allowed_errors: + if e.response["Error"]["Code"] in allowed_errors: context.error = e return else: @@ -277,10 +280,7 @@ def step_impl(context, stack_name, change_set_name): @then('the change sets for stack "{stack_name}" are listed') def step_impl(context, stack_name): full_name = get_cloudformation_stack_name(context, stack_name) - response = retry_boto_call( - context.client.list_change_sets, - StackName=full_name - ) + response = retry_boto_call(context.client.list_change_sets, StackName=full_name) for output in context.output: assert output == {stack_name: response.get("Summaries", {})} @@ -297,7 +297,7 @@ def step_impl(context, change_set_name, stack_name): response = retry_boto_call( context.client.describe_change_set, StackName=full_name, - ChangeSetName=change_set_name + ChangeSetName=change_set_name, ) del response["ResponseMetadata"] @@ -311,10 +311,7 @@ def step_impl(context, change_set_name, stack_name): @then('stack "{stack_name}" was updated with change set "{change_set_name}"') def step_impl(context, stack_name, change_set_name): full_name = get_cloudformation_stack_name(context, stack_name) - response = retry_boto_call( - context.client.describe_stacks, - StackName=full_name - ) + response = retry_boto_call(context.client.describe_stacks, StackName=full_name) change_set_id = response["Stacks"][0]["ChangeSetId"] stack_status = response["Stacks"][0]["StackStatus"] @@ -328,10 +325,10 @@ def get_change_set_status(context, stack_name, change_set_name): response = retry_boto_call( context.client.describe_change_set, ChangeSetName=change_set_name, - StackName=stack_name + StackName=stack_name, ) except ClientError as e: - if e.response['Error']['Code'] == 'ChangeSetNotFound': + if e.response["Error"]["Code"] == "ChangeSetNotFound": return None else: raise e diff --git a/integration-tests/steps/drift.py b/integration-tests/steps/drift.py new file mode 100644 index 000000000..09d411079 --- /dev/null +++ b/integration-tests/steps/drift.py @@ -0,0 +1,75 @@ +from behave import * +from helpers import get_cloudformation_stack_name + +import boto3 + +from sceptre.plan.plan import SceptrePlan +from sceptre.context import SceptreContext + + +@given('a topic configuration in stack "{stack_name}" has drifted') +def step_impl(context, stack_name): + full_name = get_cloudformation_stack_name(context, stack_name) + topic_arn = _get_output("TopicName", full_name) + client = boto3.client("sns") + client.set_topic_attributes( + TopicArn=topic_arn, AttributeName="DisplayName", AttributeValue="WrongName" + ) + + +def _get_output(output_name, stack_name): + client = boto3.client("cloudformation") + response = client.describe_stacks(StackName=stack_name) + for output in response["Stacks"][0]["Outputs"]: + if output["OutputKey"] == output_name: + return output["OutputValue"] + + +@when('the user detects drift on stack "{stack_name}"') +def step_impl(context, stack_name): + sceptre_context = SceptreContext( + command_path=stack_name + ".yaml", project_path=context.sceptre_dir + ) + sceptre_plan = SceptrePlan(sceptre_context) + values = sceptre_plan.drift_detect().values() + context.output = list(values) + + +@when('the user shows drift on stack "{stack_name}"') +def step_impl(context, stack_name): + sceptre_context = SceptreContext( + command_path=stack_name + ".yaml", project_path=context.sceptre_dir + ) + sceptre_plan = SceptrePlan(sceptre_context) + values = sceptre_plan.drift_show().values() + context.output = list(values) + + +@when('the user detects drift on stack_group "{stack_group_name}"') +def step_impl(context, stack_group_name): + sceptre_context = SceptreContext( + command_path=stack_group_name, project_path=context.sceptre_dir + ) + sceptre_plan = SceptrePlan(sceptre_context) + values = sceptre_plan.drift_detect().values() + context.output = list(values) + + +@then('stack drift status is "{desired_status}"') +def step_impl(context, desired_status): + assert context.output[0]["StackDriftStatus"] == desired_status + + +@then('stack resource drift status is "{desired_status}"') +def step_impl(context, desired_status): + assert ( + context.output[0][1]["StackResourceDrifts"][0]["StackResourceDriftStatus"] + == desired_status + ) + + +@then('stack_group drift statuses are each one of "{statuses}"') +def step_impl(context, statuses): + status_list = [status.strip() for status in statuses.split(",")] + for output in context.output: + assert output["StackDriftStatus"] in status_list diff --git a/integration-tests/steps/helpers.py b/integration-tests/steps/helpers.py index 792be1ff2..587aa1267 100644 --- a/integration-tests/steps/helpers.py +++ b/integration-tests/steps/helpers.py @@ -13,24 +13,24 @@ @then('the user is told "{message}"') def step_impl(context, message): if message == "stack does not exist": - msg = context.error.response['Error']['Message'] + msg = context.error.response["Error"]["Message"] assert msg.endswith("does not exist") elif message == "change set does not exist": - msg = context.error.response['Error']['Message'] + msg = context.error.response["Error"]["Message"] assert msg.endswith("does not exist") elif message == "the template is valid": for stack, status in context.response.items(): assert status["ResponseMetadata"]["HTTPStatusCode"] == 200 elif message == "the template is malformed": - msg = context.error.response['Error']['Message'] + msg = context.error.response["Error"]["Message"] assert msg.startswith("Template format error") else: raise Exception("Step has incorrect message") -@then('no exception is raised') +@then("no exception is raised") def step_impl(context): - assert (context.error is None) + assert context.error is None @then('a "{exception_type}" is raised') @@ -53,11 +53,9 @@ def step_impl(context, exception_type): @given('stack_group "{stack_group}" has AWS config "{config}" set') def step_impl(context, stack_group, config): - config_path = os.path.join( - context.sceptre_dir, "config", stack_group, config - ) + config_path = os.path.join(context.sceptre_dir, "config", stack_group, config) - os.environ['AWS_CONFIG_FILE'] = config_path + os.environ["AWS_CONFIG_FILE"] = config_path def read_template_file(context, template_name): @@ -67,13 +65,11 @@ def read_template_file(context, template_name): def get_cloudformation_stack_name(context, stack_name): - return "-".join( - [context.project_code, stack_name.replace("/", "-")] - ) + return "-".join([context.project_code, stack_name.replace("/", "-")]) def retry_boto_call(func, *args, **kwargs): - delay = 2 + delay = 5 max_retries = 150 attempts = 0 while attempts < max_retries: @@ -82,7 +78,7 @@ def retry_boto_call(func, *args, **kwargs): response = func(*args, **kwargs) return response except ClientError as e: - if e.response['Error']['Code'] == 'Throttling': + if e.response["Error"]["Code"] == "Throttling": time.sleep(delay) else: raise e diff --git a/integration-tests/steps/project_dependencies.py b/integration-tests/steps/project_dependencies.py new file mode 100644 index 000000000..f96cdca96 --- /dev/null +++ b/integration-tests/steps/project_dependencies.py @@ -0,0 +1,131 @@ +from itertools import chain +from typing import ContextManager, Dict + +import boto3 +from behave import given, then, when +from behave.runner import Context + +from helpers import get_cloudformation_stack_name, retry_boto_call +from sceptre.context import SceptreContext +from sceptre.plan.plan import SceptrePlan +from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error + + +@given('all files in template bucket for stack "{stack_name}" are deleted at cleanup') +def step_impl(context: Context, stack_name): + """Add this as a given to ensure that the template bucket is cleaned up before we attempt to + delete it; Otherwise, it will fail since you can't delete a bucket with objects in it. + """ + context.add_cleanup( + cleanup_template_files_in_bucket, context.sceptre_dir, stack_name + ) + + +@given("placeholders are allowed") +def step_impl(context: Context): + placeholder_context = use_resolver_placeholders_on_error() + placeholder_context.__enter__() + context.add_cleanup(exit_placeholder_context, placeholder_context) + + +@when('the user validates stack_group "{group}"') +def step_impl(context: Context, group): + sceptre_context = SceptreContext( + command_path=group, project_path=context.sceptre_dir + ) + plan = SceptrePlan(sceptre_context) + result = plan.validate() + context.response = result + + +@then('the template for stack "{stack_name}" has been uploaded') +def step_impl(context: Context, stack_name): + sceptre_context = SceptreContext( + command_path=stack_name + ".yaml", project_path=context.sceptre_dir + ) + plan = SceptrePlan(sceptre_context) + buckets = get_template_buckets(plan) + assert len(buckets) > 0 + filtered_objects = list( + chain.from_iterable( + bucket.objects.filter(Prefix=stack_name) for bucket in buckets + ) + ) + + assert len(filtered_objects) == len(plan.command_stacks) + for stack in plan.command_stacks: + for obj in filtered_objects: + if obj.key.startswith(stack.name): + s3_template = obj.get()["Body"].read().decode("utf-8") + expected = stack.template.body + assert s3_template == expected + break + else: + assert False, "Could not found uploaded template" + + +@then( + 'the stack "{resource_stack_name}" has a notification defined by stack "{topic_stack_name}"' +) +def step_impl(context, resource_stack_name, topic_stack_name): + topic_stack_resources = get_stack_resources(context, topic_stack_name) + topic = topic_stack_resources[0]["PhysicalResourceId"] + resource_stack = describe_stack(context, resource_stack_name) + notification_arns = resource_stack["NotificationARNs"] + assert topic in notification_arns + + +@then('the tag "{key}" for stack "{stack_name}" is "{value}"') +def step_impl(context, key, stack_name, value): + stack_tags = get_stack_tags(context, stack_name) + result = stack_tags[key] + assert result == value + + +@then('the tag "{key}" for stack "{stack_name}" does not exist') +def step_impl(context, key, stack_name): + stack_tags = get_stack_tags(context, stack_name) + assert key not in stack_tags + + +def cleanup_template_files_in_bucket(sceptre_dir, stack_name): + sceptre_context = SceptreContext( + command_path=stack_name + ".yaml", project_path=sceptre_dir + ) + plan = SceptrePlan(sceptre_context) + buckets = get_template_buckets(plan) + for bucket in buckets: + bucket.objects.delete() + + +def get_template_buckets(plan: SceptrePlan): + s3_resource = boto3.resource("s3") + return [ + s3_resource.Bucket(stack.template_bucket_name) + for stack in plan.command_stacks + if stack.template_bucket_name is not None + ] + + +def get_stack_resources(context, stack_name): + cf_stack_name = get_cloudformation_stack_name(context, stack_name) + resources = retry_boto_call( + context.client.describe_stack_resources, StackName=cf_stack_name + ) + return resources["StackResources"] + + +def get_stack_tags(context, stack_name) -> Dict[str, str]: + description = describe_stack(context, stack_name) + tags = {tag["Key"]: tag["Value"] for tag in description["Tags"]} + return tags + + +def describe_stack(context, stack_name) -> dict: + cf_stack_name = get_cloudformation_stack_name(context, stack_name) + response = retry_boto_call(context.client.describe_stacks, StackName=cf_stack_name) + return response["Stacks"][0] + + +def exit_placeholder_context(placeholder_context: ContextManager): + placeholder_context.__exit__(None, None, None) diff --git a/integration-tests/steps/stack_groups.py b/integration-tests/steps/stack_groups.py index 5d22c4ed6..b6ef69b4d 100644 --- a/integration-tests/steps/stack_groups.py +++ b/integration-tests/steps/stack_groups.py @@ -1,11 +1,17 @@ -from behave import * -import os import time -from sceptre.plan.plan import SceptrePlan -from sceptre.context import SceptreContext +from pathlib import Path + +from behave import * from botocore.exceptions import ClientError -from helpers import read_template_file, get_cloudformation_stack_name -from helpers import retry_boto_call + +from helpers import read_template_file, get_cloudformation_stack_name, retry_boto_call +from sceptre.cli.launch import Launcher +from sceptre.cli.prune import PATH_FOR_WHOLE_PROJECT, Pruner +from sceptre.context import SceptreContext +from sceptre.diffing.diff_writer import DeepDiffWriter +from sceptre.diffing.stack_differ import DeepDiffStackDiffer, DifflibStackDiffer +from sceptre.helpers import sceptreise_path +from sceptre.plan.plan import SceptrePlan from stacks import wait_for_final_state from templates import set_template_path @@ -43,32 +49,36 @@ def step_impl(context, stack_group_name, status): @when('the user launches stack_group "{stack_group_name}"') def step_impl(context, stack_group_name): - sceptre_context = SceptreContext( - command_path=stack_group_name, - project_path=context.sceptre_dir - ) + launch_stack_group(context, stack_group_name) - sceptre_plan = SceptrePlan(sceptre_context) - sceptre_plan.launch() + +@when('the user launches stack_group "{stack_group_name}" with --prune') +def step_impl(context, stack_group_name): + launch_stack_group(context, stack_group_name, True) @when('the user launches stack_group "{stack_group_name}" with ignore dependencies') def step_impl(context, stack_group_name): + launch_stack_group(context, stack_group_name, False, True) + + +def launch_stack_group( + context, stack_group_name, prune=False, ignore_dependencies=False +): sceptre_context = SceptreContext( command_path=stack_group_name, project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=ignore_dependencies, ) - sceptre_plan = SceptrePlan(sceptre_context) - sceptre_plan.launch() + launcher = Launcher(sceptre_context) + launcher.launch(prune) @when('the user deletes stack_group "{stack_group_name}"') def step_impl(context, stack_group_name): sceptre_context = SceptreContext( - command_path=stack_group_name, - project_path=context.sceptre_dir + command_path=stack_group_name, project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) @@ -80,7 +90,7 @@ def step_impl(context, stack_group_name): sceptre_context = SceptreContext( command_path=stack_group_name, project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) @@ -90,8 +100,7 @@ def step_impl(context, stack_group_name): @when('the user describes stack_group "{stack_group_name}"') def step_impl(context, stack_group_name): sceptre_context = SceptreContext( - command_path=stack_group_name, - project_path=context.sceptre_dir + command_path=stack_group_name, project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) @@ -103,8 +112,8 @@ def step_impl(context, stack_group_name): for response in responses.values(): if response is None: continue - for stack in response['Stacks']: - cfn_stacks[stack['StackName']] = stack['StackStatus'] + for stack in response["Stacks"]: + cfn_stacks[stack["StackName"]] = stack["StackStatus"] context.response = [ {short_name: cfn_stacks[full_name]} @@ -118,7 +127,7 @@ def step_impl(context, stack_group_name): sceptre_context = SceptreContext( command_path=stack_group_name, project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) @@ -130,8 +139,8 @@ def step_impl(context, stack_group_name): for response in responses.values(): if response is None: continue - for stack in response['Stacks']: - cfn_stacks[stack['StackName']] = stack['StackStatus'] + for stack in response["Stacks"]: + cfn_stacks[stack["StackName"]] = stack["StackStatus"] context.response = [ {short_name: cfn_stacks[full_name]} @@ -143,20 +152,21 @@ def step_impl(context, stack_group_name): @when('the user describes resources in stack_group "{stack_group_name}"') def step_impl(context, stack_group_name): sceptre_context = SceptreContext( - command_path=stack_group_name, - project_path=context.sceptre_dir + command_path=stack_group_name, project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) context.response = sceptre_plan.describe_resources().values() -@when('the user describes resources in stack_group "{stack_group_name}" with ignore dependencies') +@when( + 'the user describes resources in stack_group "{stack_group_name}" with ignore dependencies' +) def step_impl(context, stack_group_name): sceptre_context = SceptreContext( command_path=stack_group_name, project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) @@ -170,7 +180,9 @@ def step_impl(context, stack_group_name, status): check_stack_status(context, full_stack_names, status) -@then('only the stacks in stack_group "{stack_group_name}", excluding dependencies are in "{status}"') +@then( + 'only the stacks in stack_group "{stack_group_name}", excluding dependencies are in "{status}"' +) def step_impl(context, stack_group_name, status): full_stack_names = get_full_stack_names(context, stack_group_name).values() @@ -192,7 +204,7 @@ def step_impl(context, stack_group_name, status): assert response in expected_response -@then('no resources are described') +@then("no resources are described") def step_impl(context): for stack_resources in context.response: stack_name = next(iter(stack_resources)) @@ -201,10 +213,10 @@ def step_impl(context): @then('stack "{stack_name}" is described as "{status}"') def step_impl(context, stack_name, status): - response = next(( - stack for stack in context.response - if stack_name in stack - ), {stack_name: 'PENDING'}) + response = next( + (stack for stack in context.response if stack_name in stack), + {stack_name: "PENDING"}, + ) assert response[stack_name] == status @@ -221,8 +233,7 @@ def step_impl(context, stack_group_name): for short_name, full_name in stacks_names.items(): time.sleep(1) response = retry_boto_call( - context.client.describe_stack_resources, - StackName=full_name + context.client.describe_stack_resources, StackName=full_name ) expected_resources[short_name] = response["StackResources"] @@ -244,7 +255,7 @@ def step_impl(context, stack_name): response = retry_boto_call( context.client.describe_stack_resources, - StackName=get_cloudformation_stack_name(context, stack_name) + StackName=get_cloudformation_stack_name(context, stack_name), ) expected_resources[stack_name] = response["StackResources"] @@ -259,13 +270,38 @@ def step_impl(context, stack_name): def step_impl(context, first_stack, second_stack): stacks = [ get_cloudformation_stack_name(context, first_stack), - get_cloudformation_stack_name(context, second_stack) + get_cloudformation_stack_name(context, second_stack), ] creation_times = get_stack_creation_times(context, stacks) assert creation_times[stacks[0]] < creation_times[stacks[1]] +@when('the user diffs stack group "{group_name}" with "{diff_type}"') +def step_impl(context, group_name, diff_type): + sceptre_context = SceptreContext( + command_path=group_name, project_path=context.sceptre_dir + ) + sceptre_plan = SceptrePlan(sceptre_context) + differ_classes = {"deepdiff": DeepDiffStackDiffer, "difflib": DifflibStackDiffer} + writer_class = {"deepdiff": DeepDiffWriter, "difflib": DeepDiffWriter} + + differ = differ_classes[diff_type]() + context.writer_class = writer_class[diff_type] + context.output = list(sceptre_plan.diff(differ).values()) + + +@when("the whole project is pruned") +def step_impl(context): + sceptre_context = SceptreContext( + command_path=PATH_FOR_WHOLE_PROJECT, + project_path=context.sceptre_dir, + ) + + pruner = Pruner(sceptre_context) + pruner.prune() + + def get_stack_creation_times(context, stacks): creation_times = {} response = retry_boto_call(context.client.describe_stacks) @@ -276,13 +312,19 @@ def get_stack_creation_times(context, stacks): def get_stack_names(context, stack_group_name): - path = os.path.join(context.sceptre_dir, "config", stack_group_name) + config_dir = Path(context.sceptre_dir) / "config" + path = config_dir / stack_group_name + stack_names = [] - for root, dirs, files in os.walk(path): - for filepath in files: - filename = os.path.splitext(filepath)[0] - if not filename == "config": - stack_names.append(os.path.join(stack_group_name, filename)) + + for child in path.rglob("*"): + if child.is_dir() or child.stem == "config": + continue + + relative_path = child.relative_to(config_dir) + stack_name = sceptreise_path(str(relative_path).replace(child.suffix, "")) + stack_names.append(stack_name) + return stack_names @@ -301,13 +343,12 @@ def create_stacks(context, stack_names): time.sleep(1) try: retry_boto_call( - context.client.create_stack, - StackName=stack_name, - TemplateBody=body + context.client.create_stack, StackName=stack_name, TemplateBody=body ) except ClientError as e: - if e.response['Error']['Code'] == 'AlreadyExistsException' \ - and e.response['Error']['Message'].endswith("already exists"): + if e.response["Error"]["Code"] == "AlreadyExistsException" and e.response[ + "Error" + ]["Message"].endswith("already exists"): pass else: raise e @@ -316,7 +357,7 @@ def create_stacks(context, stack_names): def delete_stacks(context, stack_names): - waiter = context.client.get_waiter('stack_delete_complete') + waiter = context.client.get_waiter("stack_delete_complete") waiter.config.delay = 5 waiter.config.max_attempts = 240 diff --git a/integration-tests/steps/stack_policies.py b/integration-tests/steps/stack_policies.py index 4610df167..52f4e44b7 100644 --- a/integration-tests/steps/stack_policies.py +++ b/integration-tests/steps/stack_policies.py @@ -14,15 +14,14 @@ def step_impl(context, stack_name, state): retry_boto_call( context.client.set_stack_policy, StackName=full_name, - StackPolicyBody=generate_stack_policy(state) + StackPolicyBody=generate_stack_policy(state), ) @when('the user unlocks stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) @@ -36,8 +35,7 @@ def step_impl(context, stack_name): @when('the user locks stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) @@ -52,19 +50,19 @@ def step_impl(context, stack_name, state): full_name = get_cloudformation_stack_name(context, stack_name) policy = get_stack_policy(context, full_name) - if state == 'not set': - assert (policy is None) + if state == "not set": + assert policy is None def get_stack_policy(context, stack_name): try: response = retry_boto_call( - context.client.get_stack_policy, - StackName=stack_name + context.client.get_stack_policy, StackName=stack_name ) except ClientError as e: - if e.response['Error']['Code'] == 'ValidationError' \ - and e.response['Error']['Message'].endswith("does not exist"): + if e.response["Error"]["Code"] == "ValidationError" and e.response["Error"][ + "Message" + ].endswith("does not exist"): return None else: raise e @@ -72,28 +70,28 @@ def get_stack_policy(context, stack_name): def generate_stack_policy(policy_type): - data = '' - if policy_type == 'allow all': + data = "" + if policy_type == "allow all": data = { "Statement": [ { "Effect": "Allow", "Action": "Update:*", "Principal": "*", - "Resource": "*" + "Resource": "*", } ] } - elif policy_type == 'deny all': + elif policy_type == "deny all": data = { "Statement": [ { "Effect": "Deny", "Action": "Update:*", "Principal": "*", - "Resource": "*" + "Resource": "*", } ] } - return json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) + return json.dumps(data, sort_keys=True, indent=4, separators=(",", ": ")) diff --git a/integration-tests/steps/stacks.py b/integration-tests/steps/stacks.py index 7b20fdb95..5d64967c3 100644 --- a/integration-tests/steps/stacks.py +++ b/integration-tests/steps/stacks.py @@ -1,3 +1,9 @@ +from ast import literal_eval +from copy import deepcopy +from io import StringIO +from pathlib import Path +from typing import Type + from behave import * import time import os @@ -7,21 +13,27 @@ from botocore.exceptions import ClientError from helpers import read_template_file, get_cloudformation_stack_name from helpers import retry_boto_call +from sceptre.cli.launch import Launcher +from sceptre.cli.prune import Pruner +from sceptre.diffing.diff_writer import DeepDiffWriter, DiffWriter +from sceptre.diffing.stack_differ import ( + DeepDiffStackDiffer, + DifflibStackDiffer, + StackDiff, +) from sceptre.plan.plan import SceptrePlan from sceptre.context import SceptreContext def set_stack_timeout(context, stack_name, stack_timeout): - config_path = os.path.join( - context.sceptre_dir, "config", stack_name + ".yaml" - ) + config_path = os.path.join(context.sceptre_dir, "config", stack_name + ".yaml") with open(config_path) as config_file: stack_config = yaml.safe_load(config_file) stack_config["stack_timeout"] = int(stack_timeout) - with open(config_path, 'w') as config_file: + with open(config_path, "w") as config_file: yaml.safe_dump(stack_config, config_file, default_flow_style=False) @@ -32,7 +44,7 @@ def step_impl(context, stack_name): if status is not None: delete_stack(context, full_name) status = get_stack_status(context, full_name) - assert (status is None) + assert status is None @given('stack "{stack_name}" does not exist in "{region_name}"') @@ -43,7 +55,7 @@ def step_impl(context, stack_name, region_name): if status is not None: delete_stack(context, full_name) status = get_stack_status(context, full_name) - assert (status is None) + assert status is None @given('stack "{stack_name}" exists in "{desired_status}" state') @@ -71,7 +83,7 @@ def step_impl(context, stack_name, desired_status): create_stack(context, full_name, body, **kwargs) status = get_stack_status(context, full_name) - assert (status == desired_status) + assert status == desired_status @given('stack "{stack_name}" exists using "{template_name}"') @@ -85,7 +97,7 @@ def step_impl(context, stack_name, template_name): create_stack(context, full_name, body) status = get_stack_status(context, full_name) - assert (status == "CREATE_COMPLETE") + assert status == "CREATE_COMPLETE" @given('the stack_timeout for stack "{stack_name}" is "{stack_timeout}"') @@ -96,11 +108,10 @@ def step_impl(context, stack_name, stack_timeout): @given('stack "{dependant_stack_name}" depends on stack "{stack_name}"') def step_impl(context, dependant_stack_name, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) plan = SceptrePlan(sceptre_context) - plan.resolve('create') + plan.resolve("create") if plan.launch_order: for stack in plan.launch_order: stk = stack.pop() @@ -112,19 +123,42 @@ def step_impl(context, dependant_stack_name, stack_name): assert False +@given('the stack config for stack "{stack_name}" has changed') +def step_impl(context, stack_name): + sceptre_context = SceptreContext( + command_path=stack_name + ".yaml", + project_path=context.sceptre_dir, + ignore_dependencies=True, + ) + yaml_file = Path(sceptre_context.full_config_path()) / f"{stack_name}.yaml" + with yaml_file.open(mode="r") as f: + loaded = yaml.load(f) + + original_config = deepcopy(loaded) + loaded["stack_tags"] = {"NewTag": "NewValue"} + dump_stack_config(yaml_file, loaded) + + context.add_cleanup(dump_stack_config, yaml_file, original_config) + + +def dump_stack_config(config_path: Path, config_dict: dict): + with config_path.open(mode="w") as f: + yaml.safe_dump(config_dict, f) + + @when('the user creates stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) try: sceptre_plan.create() except ClientError as e: - if e.response['Error']['Code'] == 'AlreadyExistsException' \ - and e.response['Error']['Message'].endswith("already exists"): + if e.response["Error"]["Code"] == "AlreadyExistsException" and e.response[ + "Error" + ]["Message"].endswith("already exists"): return else: raise e @@ -133,17 +167,18 @@ def step_impl(context, stack_name): @when('the user creates stack "{stack_name}" with ignore dependencies') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) try: sceptre_plan.create() except ClientError as e: - if e.response['Error']['Code'] == 'AlreadyExistsException' \ - and e.response['Error']['Message'].endswith("already exists"): + if e.response["Error"]["Code"] == "AlreadyExistsException" and e.response[ + "Error" + ]["Message"].endswith("already exists"): return else: raise e @@ -152,17 +187,17 @@ def step_impl(context, stack_name): @when('the user updates stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) try: sceptre_plan.update() except ClientError as e: - message = e.response['Error']['Message'] - if e.response['Error']['Code'] == 'ValidationError' \ - and message.endswith("does not exist"): + message = e.response["Error"]["Message"] + if e.response["Error"]["Code"] == "ValidationError" and message.endswith( + "does not exist" + ): return else: raise e @@ -171,18 +206,19 @@ def step_impl(context, stack_name): @when('the user updates stack "{stack_name}" with ignore dependencies') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) try: sceptre_plan.update() except ClientError as e: - message = e.response['Error']['Message'] - if e.response['Error']['Code'] == 'ValidationError' \ - and message.endswith("does not exist"): + message = e.response["Error"]["Message"] + if e.response["Error"]["Code"] == "ValidationError" and message.endswith( + "does not exist" + ): return else: raise e @@ -191,18 +227,20 @@ def step_impl(context, stack_name): @when('the user deletes stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", + project_path=context.sceptre_dir, + full_scan=True, ) sceptre_plan = SceptrePlan(sceptre_context) - sceptre_plan.resolve(command='delete', reverse=True) + sceptre_plan.resolve(command="delete", reverse=True) try: sceptre_plan.delete() except ClientError as e: - if e.response['Error']['Code'] == 'ValidationError' \ - and e.response['Error']['Message'].endswith("does not exist"): + if e.response["Error"]["Code"] == "ValidationError" and e.response["Error"][ + "Message" + ].endswith("does not exist"): return else: raise e @@ -211,19 +249,21 @@ def step_impl(context, stack_name): @when('the user deletes stack "{stack_name}" with ignore dependencies') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, + full_scan=True, ) sceptre_plan = SceptrePlan(sceptre_context) - sceptre_plan.resolve(command='delete', reverse=True) + sceptre_plan.resolve(command="delete", reverse=True) try: sceptre_plan.delete() except ClientError as e: - if e.response['Error']['Code'] == 'ValidationError' \ - and e.response['Error']['Message'].endswith("does not exist"): + if e.response["Error"]["Code"] == "ValidationError" and e.response["Error"][ + "Message" + ].endswith("does not exist"): return else: raise e @@ -231,97 +271,121 @@ def step_impl(context, stack_name): @when('the user launches stack "{stack_name}"') def step_impl(context, stack_name): + launch_stack(context, stack_name) + + +@when('the user launches stack "{stack_name}" with --prune') +def step_impl(context, stack_name): + launch_stack(context, stack_name, True) + + +@when('command path "{path}" is pruned') +def step_impl(context, path): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=path, + project_path=context.sceptre_dir, ) - sceptre_plan = SceptrePlan(sceptre_context) - - try: - sceptre_plan.launch() - except Exception as e: - context.error = e + pruner = Pruner(sceptre_context) + pruner.prune() -@when('the user launches stack "{stack_name}" with ignore dependencies') -def step_impl(context, stack_name): +def launch_stack(context, stack_name, prune=False, ignore_dependencies=False): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=ignore_dependencies, ) - sceptre_plan = SceptrePlan(sceptre_context) + launcher = Launcher(sceptre_context) try: - sceptre_plan.launch() + launcher.launch(prune) except Exception as e: context.error = e +@when('the user launches stack "{stack_name}" with ignore dependencies') +def step_impl(context, stack_name): + launch_stack(context, stack_name, False, True) + + @when('the user describes the resources of stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) context.output = list(sceptre_plan.describe_resources().values()) -@when('the user describes the resources of stack "{stack_name}" with ignore dependencies') +@when( + 'the user describes the resources of stack "{stack_name}" with ignore dependencies' +) def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) context.output = list(sceptre_plan.describe_resources().values()) -@then( - 'stack "{stack_name}" in "{region_name}" ' - 'exists in "{desired_status}" state' -) +@when('the user diffs stack "{stack_name}" with "{diff_type}"') +def step_impl(context, stack_name, diff_type): + sceptre_context = SceptreContext( + command_path=stack_name + ".yaml", project_path=context.sceptre_dir + ) + sceptre_plan = SceptrePlan(sceptre_context) + differ_classes = {"deepdiff": DeepDiffStackDiffer, "difflib": DifflibStackDiffer} + writer_class = {"deepdiff": DeepDiffWriter, "difflib": DeepDiffWriter} + + differ = differ_classes[diff_type]() + context.writer_class = writer_class[diff_type] + context.output = list(sceptre_plan.diff(differ).values()) + + +@then('stack "{stack_name}" in "{region_name}" ' 'exists in "{desired_status}" state') def step_impl(context, stack_name, region_name, desired_status): with region(region_name): full_name = get_cloudformation_stack_name(context, stack_name) status = get_stack_status(context, full_name, region_name) - assert (status == desired_status) + assert status == desired_status @then('stack "{stack_name}" exists in "{desired_status}" state') def step_impl(context, stack_name, desired_status): full_name = get_cloudformation_stack_name(context, stack_name) - sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir - ) - - sceptre_plan = SceptrePlan(sceptre_context) - status = sceptre_plan.get_status() status = get_stack_status(context, full_name) - assert (status == desired_status) + assert status == desired_status + + +@then('stack "{stack_name}" has "{tag_name}" tag with "{desired_tag_value}" value') +def step_impl(context, stack_name, tag_name, desired_tag_value): + full_name = get_cloudformation_stack_name(context, stack_name) + + tags = get_stack_tags(context, full_name) + tag = next((tag for tag in tags if tag["Key"] == tag_name), {"Value": None}) + + assert tag["Value"] == desired_tag_value @then('stack "{stack_name}" does not exist') def step_impl(context, stack_name): full_name = get_cloudformation_stack_name(context, stack_name) status = get_stack_status(context, full_name) - assert (status is None) + assert status is None @then('the resources of stack "{stack_name}" are described') def step_impl(context, stack_name): full_name = get_cloudformation_stack_name(context, stack_name) response = retry_boto_call( - context.client.describe_stack_resources, - StackName=full_name + context.client.describe_stack_resources, StackName=full_name ) properties = {"LogicalResourceId", "PhysicalResourceId"} formatted_response = [ @@ -332,27 +396,73 @@ def step_impl(context, stack_name): assert [{stack_name: formatted_response}] == context.output -@then('stack "{stack_name}" does not exist and stack "{dependant_stack_name}" exists in "{desired_state}"') +@then( + 'stack "{stack_name}" does not exist and stack "{dependant_stack_name}" exists in "{desired_state}"' +) def step_impl(context, stack_name, dependant_stack_name, desired_state): full_name = get_cloudformation_stack_name(context, stack_name) status = get_stack_status(context, full_name) - assert (status is None) + assert status is None dep_full_name = get_cloudformation_stack_name(context, dependant_stack_name) sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) dep_status = sceptre_plan.get_status() dep_status = get_stack_status(context, dep_full_name) - assert (dep_status == desired_state) + assert dep_status == desired_state + + +@then('a diff is returned with "{attribute}" = "{value}"') +def step_impl(context, attribute, value): + for diff in context.output: + expected_value = literal_eval(value) + actual_value = getattr(diff, attribute) + assert actual_value == expected_value + + +@then('a diff is returned with {a_or_no} "{kind}" difference') +def step_impl(context, a_or_no, kind): + if a_or_no == "a": + test_value = True + elif a_or_no == "no": + test_value = False + else: + raise ValueError('Only "a" or "no" accepted in this condition') + + writer_class: Type[DiffWriter] = context.writer_class + difference_property = f"has_{kind}_difference" + + for diff in context.output: + diff: StackDiff + writer = writer_class(diff, StringIO(), "yaml") + assert getattr(writer, difference_property) is test_value + + +def get_stack_tags(context, stack_name, region_name=None): + if region_name is not None: + stack = boto3.resource("cloudformation", region_name=region_name).Stack + else: + stack = context.cloudformation.Stack + + try: + stack = retry_boto_call(stack, stack_name) + retry_boto_call(stack.load) + return stack.tags + except ClientError as e: + if e.response["Error"]["Code"] == "ValidationError" and e.response["Error"][ + "Message" + ].endswith("does not exist"): + return None + else: + raise e def get_stack_status(context, stack_name, region_name=None): if region_name is not None: - Stack = boto3.resource('cloudformation', region_name=region_name).Stack + Stack = boto3.resource("cloudformation", region_name=region_name).Stack else: Stack = context.cloudformation.Stack @@ -361,8 +471,9 @@ def get_stack_status(context, stack_name, region_name=None): retry_boto_call(stack.load) return stack.stack_status except ClientError as e: - if e.response['Error']['Code'] == 'ValidationError' \ - and e.response['Error']['Message'].endswith("does not exist"): + if e.response["Error"]["Code"] == "ValidationError" and e.response["Error"][ + "Message" + ].endswith("does not exist"): return None else: raise e @@ -371,7 +482,14 @@ def get_stack_status(context, stack_name, region_name=None): def create_stack(context, stack_name, body, **kwargs): retry_boto_call( context.client.create_stack, - StackName=stack_name, TemplateBody=body, **kwargs + StackName=stack_name, + TemplateBody=body, + **kwargs, + Capabilities=[ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], ) wait_for_final_state(context, stack_name) @@ -388,8 +506,8 @@ def delete_stack(context, stack_name): stack = retry_boto_call(context.cloudformation.Stack, stack_name) retry_boto_call(stack.delete) - waiter = context.client.get_waiter('stack_delete_complete') - waiter.config.delay = 4 + waiter = context.client.get_waiter("stack_delete_complete") + waiter.config.delay = 5 waiter.config.max_attempts = 240 waiter.wait(StackName=stack_name) diff --git a/integration-tests/steps/templates.py b/integration-tests/steps/templates.py index a3a242da2..98ef3473d 100644 --- a/integration-tests/steps/templates.py +++ b/integration-tests/steps/templates.py @@ -1,9 +1,10 @@ +import boto3 from behave import * import os -import imp import yaml from botocore.exceptions import ClientError +from importlib.machinery import SourceFileLoader from sceptre.plan.plan import SceptrePlan from sceptre.context import SceptreContext from sceptre.cli.helpers import CfnYamlLoader @@ -11,23 +12,31 @@ def set_template_path(context, stack_name, template_name): sceptre_context = SceptreContext( - command_path=stack_name + ".yaml", - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) config_path = sceptre_context.full_config_path() template_path = os.path.join( - sceptre_context.project_path, - sceptre_context.templates_path, - template_name + sceptre_context.project_path, sceptre_context.templates_path, template_name ) - with open(os.path.join(config_path, stack_name + '.yaml')) as config_file: + with open(os.path.join(config_path, stack_name + ".yaml")) as config_file: stack_config = yaml.safe_load(config_file) - stack_config["template_path"] = template_path - - with open(os.path.join(config_path, stack_name + '.yaml'), 'w') as config_file: + if "template_path" in stack_config: + stack_config["template_path"] = template_path + if "template" in stack_config: + stack_config["template"]["type"] = stack_config["template"].get("type", "file") + template_handler_type = stack_config["template"]["type"] + if template_handler_type.lower() == "s3": + segments = stack_config["template"]["path"].split("/") + bucket = context.TEST_ARTIFACT_BUCKET_NAME + key = "/".join(segments[1:]) + stack_config["template"]["path"] = f"{bucket}/{key}" + else: + stack_config["template"]["path"] = template_path + + with open(os.path.join(config_path, stack_name + ".yaml"), "w") as config_file: yaml.safe_dump(stack_config, config_file, default_flow_style=False) @@ -39,8 +48,7 @@ def step_impl(context, stack_name, template_name): @when('the user validates the template for stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) sceptre_plan = SceptrePlan(sceptre_context) @@ -51,12 +59,14 @@ def step_impl(context, stack_name): context.error = e -@when('the user validates the template for stack "{stack_name}" with ignore dependencies') +@when( + 'the user validates the template for stack "{stack_name}" with ignore dependencies' +) def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) @@ -70,9 +80,25 @@ def step_impl(context, stack_name): @when('the user generates the template for stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=stack_name + ".yaml", project_path=context.sceptre_dir ) + + config_path = sceptre_context.full_config_path() + template_path = sceptre_context.full_templates_path() + with open(os.path.join(config_path, stack_name + ".yaml")) as config_file: + stack_config = yaml.safe_load(config_file) + + if "template" in stack_config and stack_config["template"]["type"].lower() == "s3": + segments = stack_config["template"]["path"].split("/") + bucket = segments[0] + key = "/".join(segments[1:]) + source_file = f"{template_path}/{segments[-1]}" + boto3.client("s3").upload_file(source_file, bucket, key) + else: + config_path = sceptre_context.full_config_path() + with open(os.path.join(config_path, stack_name + ".yaml")) as config_file: + stack_config = yaml.safe_load(config_file) + sceptre_plan = SceptrePlan(sceptre_context) try: context.output = sceptre_plan.generate() @@ -80,12 +106,14 @@ def step_impl(context, stack_name): context.error = e -@when('the user generates the template for stack "{stack_name}" with ignore dependencies') +@when( + 'the user generates the template for stack "{stack_name}" with ignore dependencies' +) def step_impl(context, stack_name): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', + command_path=stack_name + ".yaml", project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=True, ) sceptre_plan = SceptrePlan(sceptre_context) try: @@ -96,23 +124,23 @@ def step_impl(context, stack_name): @then('the output is the same as the contents of "{filename}" template') def step_impl(context, filename): + filepath = os.path.join(context.sceptre_dir, "templates", filename) - filepath = os.path.join( - context.sceptre_dir, "templates", filename - ) with open(filepath) as template: body = template.read() for template in context.output.values(): - assert yaml.load(body, Loader=CfnYamlLoader) == yaml.load(template, CfnYamlLoader) + assert yaml.load(body, Loader=CfnYamlLoader) == yaml.load( + template, CfnYamlLoader + ) -@then('the output is the same as the string returned by "{filename}"') +@then('the output is the same as the contents returned by "{filename}"') def step_impl(context, filename): - filepath = os.path.join( - context.sceptre_dir, "templates", filename - ) + filepath = os.path.join(context.sceptre_dir, "templates", filename) - module = imp.load_source("template", filepath) + module = SourceFileLoader("template", filepath).load_module() body = module.sceptre_handler({}) for template in context.output.values(): - assert body == template + assert yaml.load(body, Loader=CfnYamlLoader) == yaml.load( + template, CfnYamlLoader + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..e25f0587b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "wheel"] + +[tool.coverage.run] +branch = true +source = ["sceptre"] + +[tool.coverage.report] +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", # Don't complain about missing debug-only code: + "def __repr__", + "if self.debug", # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", # Don't complain about mypy-specific code + "if TYPE_CHECKING:", +] +ignore_errors = true +fail_under = 90.0 +show_missing = true + +[tool.pytest.ini_options] +# The "--cov" option will disable the pycharm debugger. To enable the debugger either +# remove this flag or add the "--no-cov" to your pycharm's pytest run configuration. +# https://github.com/pytest-dev/pytest-cov/issues/131 +# https://stackoverflow.com/questions/34870962/how-to-debug-py-test-in-pycharm-when-coverage-is-enabled +addopts = "-s --cov --cov-report term --cov-report term-missing --cov-report xml" diff --git a/requirements/dev.txt b/requirements/dev.txt index 46b2dc649..26819dce7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,23 +1,22 @@ -behave==1.2.5 -bumpversion==0.5.3 -coverage==4.4.2 -flake8==3.5.0 -flake8-per-file-ignores==0.4 -freezegun==0.3.12 -Jinja2>=2.8,<3 -mock>=2.0.0,<2.1.0 -moto==1.3.8 -networkx==2.1 -packaging==16.8 -pygments==2.2.0 -pytest-runner>=3.0.0,<3.1.0 -pytest>=3.2.0,<3.3.0 -readme-renderer>=24.0 -setuptools>=40.6.2 -Sphinx==1.6.5 -sphinx-click==2.0.1 -sphinx-rtd-theme==0.4.3 -tox>=2.9.1,<3.0.0 -troposphere>=2.0.0,<2.1.0 -twine>=1.12.1 -wheel==0.32.3 +behave>=1.2.5,<2.0.0 +bumpversion>=0.5.3,<0.6.0 +coverage[toml]>=5.1,<6.0 +flake8>=3.9.1,<4.0.0 +pre-commit>=2.12.0,<2.13 +freezegun>=0.3.12,<0.4.0 +moto>=3.0,<4 +pygments>=2.2.0,<3.0.0 +pytest>=6.2.0,<7.0.0 +pytest-cov>=2.11.1,<3.0.0 +pytest-sugar>=0.9.4,<1.0.0 +readme-renderer>=24.0,<25.0 +requests-mock>=1.9.3,<2.0 +setuptools==65.5.1 +Sphinx>=1.6.5,<=5.1.1 +sphinx-click>=2.0.1,<4.0.0 +sphinx-rtd-theme==0.5.2 +sphinx-autodoc-typehints==1.19.2 +docutils<0.17 # temporary fix for sphinx-rtd-theme==0.5.2, it depends on docutils<0.17 +tox>=3.23.0,<4.0.0 +twine>=1.12.1,<2.0.0 +wheel==0.38.1 diff --git a/requirements/prod.txt b/requirements/prod.txt index 3e1894eb7..a6ab13ca3 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,7 +1,15 @@ boto3>=1.3.0,<2 -click==7.0 -colorama==0.3.9 -Jinja2>=2.8,<3 -networkx==2.1 +click>=7.0,<9.0 +colorama>=0.2.5,<0.4.4 +cfn-flip>=1.2.3,<2.0 +deepdiff>=5.5.0,<6.0 +deprecation>=2.0.0,<3.0 +Jinja2>=3.0,<4 +jsonschema>=3.2,<3.3 +networkx>=2.6,<2.7 +packaging>=16.8,<22.0 PyYaml>=5.1,<6.0 -typing>=3.7,<3.8 +sceptre-cmd-resolver>=1.1.3,<3 +sceptre-file-resolver>=1.0.4,<2 +six>=1.11.0,<2.0.0 +troposphere>=4,<5 diff --git a/sceptre/__init__.py b/sceptre/__init__.py index 99b6b0a81..ba240b86e 100644 --- a/sceptre/__init__.py +++ b/sceptre/__init__.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- import logging +import sys import warnings -__author__ = 'Cloudreach' -__email__ = 'sceptre@cloudreach.com' -__version__ = '2.2.1' +__author__ = "Cloudreach" +__email__ = "sceptre@cloudreach.com" +__version__ = "4.0.2" # Set up logging to ``/dev/null`` like a library is supposed to. @@ -16,7 +17,7 @@ def emit(self, record): pass -with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) +if not sys.warnoptions: + warnings.filterwarnings("default", category=DeprecationWarning, module="sceptre") -logging.getLogger('sceptre').addHandler(NullHandler()) +logging.getLogger("sceptre").addHandler(NullHandler()) diff --git a/sceptre/cli/__init__.py b/sceptre/cli/__init__.py index 172915783..ac8790640 100644 --- a/sceptre/cli/__init__.py +++ b/sceptre/cli/__init__.py @@ -8,26 +8,31 @@ import os -import warnings - import click import colorama -import yaml from sceptre import __version__ -from sceptre.cli.new import new_group from sceptre.cli.create import create_command -from sceptre.cli.update import update_command from sceptre.cli.delete import delete_command -from sceptre.cli.launch import launch_command -from sceptre.cli.execute import execute_command from sceptre.cli.describe import describe_group +from sceptre.cli.diff import diff_command +from sceptre.cli.drift import drift_group +from sceptre.cli.dump import dump_group +from sceptre.cli.execute import execute_command +from sceptre.cli.helpers import catch_exceptions, setup_vars +from sceptre.cli.launch import launch_command from sceptre.cli.list import list_group +from sceptre.cli.new import new_group from sceptre.cli.policy import set_policy_command +from sceptre.cli.prune import prune_command from sceptre.cli.status import status_command -from sceptre.cli.template import (validate_command, generate_command, - estimate_cost_command) -from sceptre.cli.helpers import setup_logging, catch_exceptions +from sceptre.cli.template import ( + validate_command, + generate_command, + estimate_cost_command, + fetch_remote_template_command, +) +from sceptre.cli.update import update_command @click.group() @@ -35,67 +40,58 @@ @click.option("--debug", is_flag=True, help="Turn on debug logging.") @click.option("--dir", "directory", help="Specify sceptre directory.") @click.option( - "--output", type=click.Choice(["text", "yaml", "json"]), default="json", - help="The formatting style for command output.") + "--output", + type=click.Choice(["text", "yaml", "json"]), + default="text", + help="The formatting style for command output.", +) @click.option("--no-colour", is_flag=True, help="Turn off output colouring.") @click.option( - "--var", multiple=True, help="A variable to template into config files.") + "--var", + multiple=True, + help="A variable to replace the value of an item in config file.", +) +@click.option( + "--var-file", + multiple=True, + type=click.File("rb"), + help="A YAML file of variables to replace the values of items in config files.", +) @click.option( - "--var-file", multiple=True, type=click.File("rb"), - help="A YAML file of variables to template into config files.") + "--ignore-dependencies", + is_flag=True, + help="Ignore dependencies when executing command.", +) @click.option( - "--ignore-dependencies", is_flag=True, help="Ignore dependencies when executing command.") + "--merge-vars", + is_flag=True, + default=False, + help="Merge variables from successive --vars and var files", +) @click.pass_context @catch_exceptions def cli( - ctx, debug, directory, output, no_colour, var, var_file, ignore_dependencies + ctx, + debug, + directory, + output, + no_colour, + var, + var_file, + ignore_dependencies, + merge_vars, ): """ Sceptre is a tool to manage your cloud native infrastructure deployments. - """ - logger = setup_logging(debug, no_colour) colorama.init() - # Enable deprecation warnings - warnings.simplefilter("always", DeprecationWarning) ctx.obj = { - "user_variables": {}, + "user_variables": setup_vars(var_file, var, merge_vars, debug, no_colour), "output_format": output, "no_colour": no_colour, "ignore_dependencies": ignore_dependencies, - "project_path": directory if directory else os.getcwd() + "project_path": directory if directory else os.getcwd(), } - if var_file: - for fh in var_file: - parsed = yaml.safe_load(fh.read()) - ctx.obj.get("user_variables").update(parsed) - - # the rest of this block is for debug purposes only - existing_keys = set(ctx.obj.get("user_variables").keys()) - new_keys = set(parsed.keys()) - overloaded_keys = existing_keys & new_keys # intersection - if overloaded_keys: - logger.debug( - "Duplicate variables encountered: {0}. " - "Using values from: {1}." - .format(", ".join(overloaded_keys), fh.name) - ) - - if var: - def update_dict(variable): - variable_key, variable_value = variable.split("=") - keys = variable_key.split(".") - - def nested_set(dic, keys, value): - for key in keys[:-1]: - dic = dic.setdefault(key, {}) - dic[keys[-1]] = value - - nested_set(ctx.obj.get("user_variables"), keys, variable_value) - - # --var options overwrite --var-file options - for variable in var: - update_dict(variable) cli.add_command(new_group) @@ -110,4 +106,9 @@ def nested_set(dic, keys, value): cli.add_command(set_policy_command) cli.add_command(status_command) cli.add_command(list_group) +cli.add_command(dump_group) cli.add_command(describe_group) +cli.add_command(fetch_remote_template_command) +cli.add_command(diff_command) +cli.add_command(drift_group) +cli.add_command(prune_command) diff --git a/sceptre/cli/create.py b/sceptre/cli/create.py index 0b717e641..205fa08cb 100644 --- a/sceptre/cli/create.py +++ b/sceptre/cli/create.py @@ -1,5 +1,6 @@ import click +from typing import Optional from sceptre.context import SceptreContext from sceptre.cli.helpers import catch_exceptions, confirmation from sceptre.plan.plan import SceptrePlan @@ -9,12 +10,15 @@ @click.command(name="create", short_help="Creates a stack or a change set.") @click.argument("path") @click.argument("change-set-name", required=False) +@click.option("-y", "--yes", is_flag=True, help="Assume yes to all questions.") @click.option( - "-y", "--yes", is_flag=True, help="Assume yes to all questions." + "--disable-rollback/--enable-rollback", + default=None, + help="Disable or enable the cloudformation automatic rollback", ) @click.pass_context @catch_exceptions -def create_command(ctx, path, change_set_name, yes): +def create_command(ctx, path, change_set_name, yes, disable_rollback: Optional[bool]): """ Creates a stack for a given config PATH. Or if CHANGE_SET_NAME is specified creates a change set for stack in PATH. @@ -26,21 +30,22 @@ def create_command(ctx, path, change_set_name, yes): :type change_set_name: str :param yes: A flag to assume yes to all questions. :type yes: bool + :param disable_rollback: A flag to disable cloudformation rollback. """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) action = "create" plan = SceptrePlan(context) if change_set_name: - confirmation(action, yes, change_set=change_set_name, - command_path=path) + confirmation(action, yes, change_set=change_set_name, command_path=path) plan.create_change_set(change_set_name) else: confirmation(action, yes, command_path=path) diff --git a/sceptre/cli/delete.py b/sceptre/cli/delete.py index 24b422d53..993f2549f 100644 --- a/sceptre/cli/delete.py +++ b/sceptre/cli/delete.py @@ -12,9 +12,7 @@ @click.command(name="delete", short_help="Deletes a stack or a change set.") @click.argument("path") @click.argument("change-set-name", required=False) -@click.option( - "-y", "--yes", is_flag=True, help="Assume yes to all questions." -) +@click.option("-y", "--yes", is_flag=True, help="Assume yes to all questions.") @click.pass_context @catch_exceptions def delete_command(ctx, path, change_set_name, yes): @@ -32,32 +30,32 @@ def delete_command(ctx, path, change_set_name, yes): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + full_scan=True, ) plan = SceptrePlan(context) - plan.resolve(command='delete', reverse=True) + plan.resolve(command="delete", reverse=True) if change_set_name: - delete_msg = "The Change Set will be delete on the following stacks, if applicable:\n" + delete_msg = ( + "The Change Set will be delete on the following stacks, if applicable:\n" + ) else: delete_msg = "The following stacks, in the following order, will be deleted:\n" - dependencies = '' - for stacks in plan.launch_order: - for stack in stacks: - dependencies += "{}{}{}\n".format(Fore.YELLOW, stack.name, Style.RESET_ALL) + dependencies = "" + for stack in plan: + dependencies += "{}{}{}\n".format(Fore.YELLOW, stack.name, Style.RESET_ALL) print(delete_msg + "{}".format(dependencies)) confirmation( - plan.delete.__name__, - yes, - change_set=change_set_name, - command_path=path + plan.delete.__name__, yes, change_set=change_set_name, command_path=path ) if change_set_name: plan.delete_change_set(change_set_name) diff --git a/sceptre/cli/describe.py b/sceptre/cli/describe.py index 5b54c8fd8..5b5048b5c 100644 --- a/sceptre/cli/describe.py +++ b/sceptre/cli/describe.py @@ -1,11 +1,7 @@ import click from sceptre.context import SceptreContext -from sceptre.cli.helpers import ( - catch_exceptions, - simplify_change_set_description, - write -) +from sceptre.cli.helpers import catch_exceptions, simplify_change_set_description, write from sceptre.plan.plan import SceptrePlan @@ -21,9 +17,7 @@ def describe_group(ctx): @describe_group.command(name="change-set") @click.argument("path") @click.argument("change-set-name") -@click.option( - "-v", "--verbose", is_flag=True, help="Display verbose output." -) +@click.option("-v", "--verbose", is_flag=True, help="Display verbose output.") @click.pass_context @catch_exceptions def describe_change_set(ctx, path, change_set_name, verbose): @@ -40,12 +34,13 @@ def describe_change_set(ctx, path, change_set_name, verbose): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), output_format=ctx.obj.get("output_format"), no_colour=ctx.obj.get("no_colour"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) @@ -72,19 +67,16 @@ def describe_policy(ctx, path): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), output_format=ctx.obj.get("output_format"), no_colour=ctx.obj.get("no_colour"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) responses = plan.get_policy() for response in responses.values(): - write( - response, - context.output_format, - context.no_colour - ) + write(response, context.output_format, context.no_colour) diff --git a/sceptre/cli/diff.py b/sceptre/cli/diff.py new file mode 100644 index 000000000..303b156c5 --- /dev/null +++ b/sceptre/cli/diff.py @@ -0,0 +1,203 @@ +import io +import sys +from logging import getLogger +from typing import Dict, TextIO, Type, Iterable + +import click +from click import Context + +from sceptre.cli.helpers import catch_exceptions +from sceptre.context import SceptreContext +from sceptre.diffing.diff_writer import ( + DeepDiffWriter, + DiffLibWriter, + ColouredDiffLibWriter, + DiffWriter, +) +from sceptre.diffing.stack_differ import ( + DeepDiffStackDiffer, + DifflibStackDiffer, + StackDiff, +) +from sceptre.helpers import null_context +from sceptre.plan.plan import SceptrePlan +from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error +from sceptre.stack import Stack + +logger = getLogger(__name__) + + +@click.command( + name="diff", + short_help="Compares deployed infrastructure with current configurations", +) +@click.option( + "-t", + "--type", + "differ", + type=click.Choice(["deepdiff", "difflib"]), + default="deepdiff", + help='The type of differ to use. Use "deepdiff" for recursive key/value comparison. "difflib" ' + 'produces a more traditional "diff" result. Defaults to deepdiff.', +) +@click.option( + "-s", + "--show-no-echo", + is_flag=True, + help="If set, will display the unmasked values of NoEcho parameters generated LOCALLY (NoEcho " + "parameters for deployed stacks will always be masked when retrieved from CloudFormation.). " + "If not set (the default), parameters identified as NoEcho on the local template will be " + "masked when presented in the diff.", +) +@click.option( + "-n", + "--no-placeholders", + is_flag=True, + help="If set, no placeholder values will be supplied for resolvers that cannot be resolved.", +) +@click.option( + "-a", + "--all", + "all_", + is_flag=True, + help=( + "If set, will perform diffing on ALL stacks, including ignored and obsolete ones; Otherwise, " + "it will diff only stacks that would be created or updated when running the launch command." + ), +) +@click.argument("path") +@click.pass_context +@catch_exceptions +def diff_command( + ctx: Context, + differ: str, + show_no_echo: bool, + no_placeholders: bool, + all_: bool, + path: str, +): + """Indicates the difference between the currently DEPLOYED stacks in the command path and + the stacks configured in Sceptre right now. This command will compare both the templates as well + as the subset of stack configurations that can be compared. By default, only stacks that would + be launched via the launch command will be diffed, but you can diff ALL stacks relevant to the + passed command path if you pass the --all flag. + + Some settings (such as sceptre_user_data) are not available in a CloudFormation stack + description, so the diff will not be indicated. Currently compared stack configurations are: + + \b + * parameters + * notifications + * cloudformation_service_role + * stack_tags + + Important: There are resolvers (notably !stack_output) that rely on other stacks + to be already deployed when they are resolved. When producing a diff on Stack Configs that have + such resolvers that point to non-deployed stacks, this presents a challenge, since this means + those resolvers cannot be resolved. This particularly applies to stack parameters and when a + stack's template uses sceptre_user_data with resolvers in it. In order to continue to be useful + when producing a diff in these conditions, this command will do the following: + + 1. If the resolver CAN be resolved, it will be resolved and the resolved value will be in the + diff results. + 2. If the resolver CANNOT be resolved, it will be replaced with a string that represents the + resolver and its arguments. For example: !stack_output my_stack.yaml::MyOutput will resolve in + the parameters to "{ !StackOutput(my_stack.yaml::MyOutput) }". + + Particularly in cases where the replaced value doesn't work in the template as the template logic + requires and causes an error, there is nothing further Sceptre can do and diffing will fail. + """ + no_colour = ctx.obj.get("no_colour") + + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + output_format=ctx.obj.get("output_format"), + no_colour=no_colour, + ) + output_format = context.output_format + plan = SceptrePlan(context) + if not all_: + filter_plan_for_launchable(plan) + + if differ == "deepdiff": + stack_differ = DeepDiffStackDiffer(show_no_echo) + writer_class = DeepDiffWriter + elif differ == "difflib": + stack_differ = DifflibStackDiffer(show_no_echo) + writer_class = DiffLibWriter if no_colour else ColouredDiffLibWriter + else: + raise ValueError(f"Unexpected differ type: {differ}") + + execution_context = ( + null_context() if no_placeholders else use_resolver_placeholders_on_error() + ) + with execution_context: + diffs: Dict[Stack, StackDiff] = plan.diff(stack_differ) + + num_stacks_with_diff = output_diffs( + diffs.values(), writer_class, sys.stdout, output_format + ) + + if num_stacks_with_diff: + logger.warning(f"{num_stacks_with_diff} stacks with differences detected.") + + +def output_diffs( + diffs: Iterable[StackDiff], + writer_class: Type[DiffWriter], + output_buffer: TextIO, + output_format: str, +) -> int: + """Outputs the diff results to the output_buffer. + + :param diffs: The differences computed + :param writer_class: The DiffWriter class to be instantiated for each StackDiff + :param output_buffer: The buffer to write the diff results to + :param output_format: The format to output the results in + :return: The number of stacks that had a difference + """ + + line_buffer = io.StringIO() + + num_stacks_with_diff = 0 + + for stack_diff in diffs: + writer = writer_class(stack_diff, line_buffer, output_format) + writer.write() + if writer.has_difference: + num_stacks_with_diff += 1 + + output_buffer_with_normalized_bar_lengths(line_buffer, output_buffer) + return num_stacks_with_diff + + +def output_buffer_with_normalized_bar_lengths( + buffer: io.StringIO, output_stream: TextIO +): + """Takes the output from a buffer and ensures that the star and line bars are the same length + across the entire buffer and that their length is the full width of longest line. + + :param buffer: The input stream to normalize bar lengths for + :param output_stream: The stream to output the normalized buffer into + """ + buffer.seek(0) + max_length = len(max(buffer, key=len)) + buffer.seek(0) + full_length_star_bar = "*" * max_length + full_length_line_bar = "-" * max_length + for line in buffer: + if DiffWriter.STAR_BAR in line: + line = line.replace(DiffWriter.STAR_BAR, full_length_star_bar) + if DiffWriter.LINE_BAR in line: + line = line.replace(DiffWriter.LINE_BAR, full_length_line_bar) + output_stream.write(line) + + +def filter_plan_for_launchable(plan: SceptrePlan): + plan.resolve(plan.diff.__name__) + plan.filter(lambda stack: not stack.ignore and not stack.obsolete) diff --git a/sceptre/cli/drift.py b/sceptre/cli/drift.py new file mode 100644 index 000000000..08b43dc3c --- /dev/null +++ b/sceptre/cli/drift.py @@ -0,0 +1,109 @@ +import click +from click import Context + +from sceptre.context import SceptreContext +from sceptre.plan.plan import SceptrePlan + +from sceptre.cli.helpers import catch_exceptions, deserialize_json_properties, write + +BAD_STATUSES = ["DETECTION_FAILED", "TIMED_OUT"] + + +@click.group(name="drift") +def drift_group(): + """ + Commands for calling drift detection. + """ + pass + + +@drift_group.command( + name="detect", short_help="Run detect stack drift on running stacks." +) +@click.argument("path") +@click.pass_context +@catch_exceptions +def drift_detect(ctx: Context, path: str): + """ + Detect stack drift and return stack drift status. + + In the event that the stack does not exist, we return + a DetectionStatus and StackDriftStatus of STACK_DOES_NOT_EXIST. + + In the event that drift detection times out, we return + a DetectionStatus and StackDriftStatus of TIMED_OUT. + + The timeout is set at 5 minutes, a value that cannot be configured. + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + output_format=ctx.obj.get("output_format"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + ) + + plan = SceptrePlan(context) + responses = plan.drift_detect() + + output_format = "json" if context.output_format == "json" else "yaml" + + exit_status = 0 + for stack, response in responses.items(): + status = response["DetectionStatus"] + if status in BAD_STATUSES: + exit_status += 1 + for key in ["Timestamp", "ResponseMetadata"]: + response.pop(key, None) + write( + {stack.external_name: deserialize_json_properties(response)}, output_format + ) + + exit(exit_status) + + +@drift_group.command(name="show", short_help="Shows stack drift on running stacks.") +@click.argument("path") +@click.option( + "-D", "--drifted", is_flag=True, default=False, help="Filter out in sync resources." +) +@click.pass_context +@catch_exceptions +def drift_show(ctx, path, drifted): + """ + Show stack drift on deployed stacks. + + In the event that the stack does not exist, we return + a StackResourceDriftStatus of STACK_DOES_NOT_EXIST. + + In the event that drift detection times out, we return + a StackResourceDriftStatus of TIMED_OUT. + + The timeout is set at 5 minutes, a value that cannot be configured. + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + output_format=ctx.obj.get("output_format"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + ) + + plan = SceptrePlan(context) + responses = plan.drift_show(drifted) + + output_format = "json" if context.output_format == "json" else "yaml" + + exit_status = 0 + for stack, (status, response) in responses.items(): + if status in BAD_STATUSES: + exit_status += 1 + write( + {stack.external_name: deserialize_json_properties(response)}, output_format + ) + + exit(exit_status) diff --git a/sceptre/cli/dump.py b/sceptre/cli/dump.py new file mode 100644 index 000000000..dee258ba5 --- /dev/null +++ b/sceptre/cli/dump.py @@ -0,0 +1,109 @@ +import logging +import click + +from sceptre.context import SceptreContext +from sceptre.cli.helpers import catch_exceptions, write +from sceptre.plan.plan import SceptrePlan +from sceptre.helpers import null_context +from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error + +logger = logging.getLogger(__name__) + + +@click.group(name="dump") +def dump_group(): + """ + Commands for dumping attributes of stacks. + """ + pass + + +@dump_group.command(name="config") +@click.argument("path") +@click.pass_context +@catch_exceptions +def dump_config(ctx, path): + """ + Dump the rendered (post-Jinja) Stack Configs. + \f + + :param path: Path to execute the command on or path to stack group + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + output_format=ctx.obj.get("output_format"), + options=ctx.obj.get("options"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + ) + plan = SceptrePlan(context) + responses = plan.dump_config() + + output = [] + for stack, config in responses.items(): + if config is None: + logger.warning(f"{stack.external_name} does not exist") + else: + output.append({stack.external_name: config}) + + output_format = "json" if context.output_format == "json" else "yaml" + + if len(output) == 1: + write(output[0][stack.external_name], output_format) + else: + for config in output: + write(config, output_format) + + +@dump_group.command(name="template") +@click.argument("path") +@click.option( + "-n", + "--no-placeholders", + is_flag=True, + help="If True, no placeholder values will be supplied for resolvers that cannot be resolved.", +) +@click.pass_context +@catch_exceptions +def dump_template(ctx, no_placeholders, path): + """ + Prints the template used for stack in PATH. + \f + + :param path: Path to execute the command on. + :type path: str + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + output_format=ctx.obj.get("output_format"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + ) + + plan = SceptrePlan(context) + + execution_context = ( + null_context() if no_placeholders else use_resolver_placeholders_on_error() + ) + with execution_context: + responses = plan.dump_template() + + output = [] + for stack, template in responses.items(): + if template is None: + logger.warning(f"{stack.external_name} does not exist") + else: + output.append({stack.external_name: template}) + + output_format = "json" if context.output_format == "json" else "yaml" + + if len(output) == 1: + write(output[0][stack.external_name], output_format) + else: + for template in output: + write(template, output_format) diff --git a/sceptre/cli/execute.py b/sceptre/cli/execute.py index f36855b7f..18f89d31c 100644 --- a/sceptre/cli/execute.py +++ b/sceptre/cli/execute.py @@ -1,5 +1,6 @@ import click +from typing import Optional from sceptre.context import SceptreContext from sceptre.cli.helpers import catch_exceptions, confirmation from sceptre.plan.plan import SceptrePlan @@ -8,12 +9,15 @@ @click.command(name="execute") @click.argument("path") @click.argument("change-set-name") +@click.option("-y", "--yes", is_flag=True, help="Assume yes to all questions.") @click.option( - "-y", "--yes", is_flag=True, help="Assume yes to all questions." + "--disable-rollback/--enable-rollback", + default=None, + help="Disable or enable the cloudformation automatic rollback", ) @click.pass_context @catch_exceptions -def execute_command(ctx, path, change_set_name, yes): +def execute_command(ctx, path, change_set_name, yes, disable_rollback: Optional[bool]): """ Executes a Change Set. \f @@ -27,10 +31,11 @@ def execute_command(ctx, path, change_set_name, yes): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) @@ -38,6 +43,6 @@ def execute_command(ctx, path, change_set_name, yes): plan.execute_change_set.__name__, yes, change_set=change_set_name, - command_path=path + command_path=path, ) plan.execute_change_set(change_set_name) diff --git a/sceptre/cli/helpers.py b/sceptre/cli/helpers.py index dd7f139b4..47efb87e2 100644 --- a/sceptre/cli/helpers.py +++ b/sceptre/cli/helpers.py @@ -1,5 +1,6 @@ import logging import sys +from itertools import cycle from functools import partial, wraps import json @@ -26,31 +27,40 @@ def catch_exceptions(func): simplified. :returns: The decorated function. """ + + def logging_level(): + logger = logging.getLogger(__name__) + return logger.getEffectiveLevel() + @wraps(func) def decorated(*args, **kwargs): """ Invokes ``func``, catches expected errors, prints the error message and - exits sceptre with a non-zero exit code. + exits sceptre with a non-zero exit code. In debug mode, the original + exception is re-raised to assist debugging. """ try: return func(*args, **kwargs) - except (SceptreException, BotoCoreError, ClientError, Boto3Error, - TemplateError) as error: + except ( + SceptreException, + BotoCoreError, + ClientError, + Boto3Error, + TemplateError, + ) as error: + if logging_level() == logging.DEBUG: + raise write(error) sys.exit(1) return decorated -def confirmation( - command, ignore, command_path, change_set=None -): +def confirmation(command, ignore, command_path, change_set=None): if not ignore: msg = "Do you want to {} ".format(command) if change_set: - msg = msg + "change set '{0}' for '{1}'".format( - change_set, command_path - ) + msg = msg + "change set '{0}' for '{1}'".format(change_set, command_path) else: msg = msg + "'{0}'".format(command_path) click.confirm(msg, abort=True) @@ -76,7 +86,7 @@ def write(var, output_format="json", no_colour=True): if output_format == "yaml": output = _generate_yaml(var) if output_format == "text": - output = var + output = _generate_text(var) if not no_colour: stack_status_colourer = StackStatusColourer() output = stack_status_colourer.colour(str(output)) @@ -105,27 +115,27 @@ def _generate_json(stream): def _generate_yaml(stream): - if isinstance(stream, list): + kwargs = {"default_flow_style": False, "explicit_start": True} + + if isinstance(stream, (list, set)): items = [] for item in stream: try: if isinstance(item, dict): - items.append( - yaml.safe_dump(item, default_flow_style=False, explicit_start=True) - ) + items.append(yaml.safe_dump(item, **kwargs)) else: items.append( - yaml.safe_dump( - yaml.load(item, Loader=CfnYamlLoader), - default_flow_style=False, explicit_start=True - ) + yaml.safe_dump(yaml.load(item, Loader=CfnYamlLoader), **kwargs) ) except Exception: print("An error occured whilst writing the YAML object.") return yaml.safe_dump( - [yaml.load(item, Loader=CfnYamlLoader) for item in items], - default_flow_style=False, explicit_start=True + [yaml.load(item, Loader=CfnYamlLoader) for item in items], **kwargs ) + + elif isinstance(stream, dict): + return yaml.dump(stream, **kwargs) + else: try: return yaml.safe_loads(stream) @@ -133,10 +143,124 @@ def _generate_yaml(stream): return stream +def _generate_text(stream): + if isinstance(stream, list): + items = [] + for item in stream: + try: + if isinstance(item, dict): + # use keys as headers, and add a blank row + if not items: + items = [["Stack"]] + items[0].extend(list(next(iter(*item.values())))) + items.append(["" for _ in range(len(items[0]))]) + for k, v in item.items(): + for r in item[k]: + row = [k] + row.extend(list(r.values())) + items.append(row) + else: + items.append(item) + except Exception: + print("An error occured writing the text object.") + col_widths = [max(len(c) for c in b) for b in zip(*items)] + rows = [] + for row in items: + rows.append( + "".join([field for field, width in zip(row, cycle(col_widths))]) + ) + return "\n".join(rows) + return stream + + +def setup_vars(var_file, var, merge_vars, debug, no_colour): + """ + Handle --var-file and --var arguments before + returning data for the user_variables as required + by the ConfigReader and SceptreContext. + + :param var_file: the var_file list. + :type var_file: List[Dict] + :param var: the var list. + :type var: List[str] + :param merge_vars: Merge instead of + overwrite duplicate keys. + :type merge_vars: bool + :param debug: debug mode. + :type debug: bool + :param no_colour: no_colour mode. + :type no_colour: bool + + :returns: data for the user_variables. + :rtype: Dict + """ + logger = setup_logging(debug, no_colour) + + return_value = {} + + def _update_dict(variable): + variable_key, variable_value = variable.split("=") + keys = variable_key.split(".") + + def _nested_set(dic, keys, value): + for key in keys[:-1]: + dic = dic.setdefault(key, {}) + dic[keys[-1]] = value + + _nested_set(return_value, keys, variable_value) + + if var_file: + for fh in var_file: + parsed = yaml.safe_load(fh.read()) or {} + + if merge_vars: + return_value = _deep_merge(parsed, return_value) + else: + return_value.update(parsed) + + # the rest of this block is for debug purposes only + existing_keys = set(return_value.keys()) + new_keys = set(parsed.keys()) + overloaded_keys = existing_keys & new_keys # intersection + + if overloaded_keys: + message = "Duplicate variables encountered: " + + if merge_vars: + message += "{0}. Using values from: {1}.".format( + ", ".join(overloaded_keys), fh.name + ) + else: + message += "{0}. Performing deep merge, {1} wins.".format( + ", ".join(overloaded_keys), fh.name + ) + + logger.debug(message) + + if var: + # --var options overwrite --var-file options, unless a dict and --merge-vars. + for variable in var: + if isinstance(variable, dict) and merge_vars: + return_value = _deep_merge(variable, return_value) + else: + _update_dict(variable) + + return return_value + + +def _deep_merge(source, destination): + for key, value in source.items(): + if isinstance(value, dict): + node = destination.setdefault(key, {}) + _deep_merge(value, node) + else: + destination[key] = value + + return destination + + def stack_status_exit_code(statuses): - if not all( - status == StackStatus.COMPLETE - for status in statuses): + if not all(status == StackStatus.COMPLETE for status in statuses): return 1 else: return 0 @@ -177,8 +301,7 @@ def setup_logging(debug, no_colour): formatter_class = logging.Formatter if no_colour else ColouredFormatter formatter = formatter_class( - fmt="[%(asctime)s] - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" + fmt="[%(asctime)s] - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" ) log_handler = logging.StreamHandler() @@ -204,7 +327,7 @@ def simplify_change_set_description(response): "ExecutionStatus", "StackName", "Status", - "StatusReason" + "StatusReason", ] desired_resource_changes = [ "Action", @@ -212,12 +335,10 @@ def simplify_change_set_description(response): "PhysicalResourceId", "Replacement", "ResourceType", - "Scope" + "Scope", ] formatted_response = { - k: v - for k, v in response.items() - if k in desired_response_items + k: v for k, v in response.items() if k in desired_response_items } formatted_response["Changes"] = [ { @@ -232,6 +353,21 @@ def simplify_change_set_description(response): return formatted_response +def deserialize_json_properties(value): + if isinstance(value, str): + is_json = (value.startswith("{") and value.endswith("}")) or ( + value.startswith("[") and value.endswith("]") + ) + if is_json: + return json.loads(value) + return value + if isinstance(value, dict): + return {key: deserialize_json_properties(val) for key, val in value.items()} + if isinstance(value, list): + return [deserialize_json_properties(item) for item in value] + return value + + class ColouredFormatter(logging.Formatter): """ ColouredFormatter add colours to all stack statuses that appear in log @@ -272,39 +408,38 @@ def default(self, item): CFN_FNS = [ - 'And', - 'Base64', - 'Cidr', - 'Equals', - 'FindInMap', - 'GetAtt', - 'GetAZs', - 'If', - 'ImportValue', - 'Join', - 'Not', - 'Or', - 'Select', - 'Split', - 'Sub', - 'Transform', + "And", + "Base64", + "Cidr", + "Equals", + "FindInMap", + "GetAtt", + "GetAZs", + "If", + "ImportValue", + "Join", + "Not", + "Or", + "Select", + "Split", + "Sub", + "Transform", ] CFN_TAGS = [ - 'Condition', - 'Ref', + "Condition", + "Ref", ] def _getatt_constructor(loader, node): if isinstance(node.value, six.text_type): - return node.value.split('.', 1) + return node.value.split(".", 1) elif isinstance(node.value, list): seq = loader.construct_sequence(node) for item in seq: if not isinstance(item, six.text_type): - raise ValueError( - "Fn::GetAtt does not support complex datastructures") + raise ValueError("Fn::GetAtt does not support complex datastructures") return seq else: raise ValueError("Fn::GetAtt only supports string or list values") @@ -312,11 +447,13 @@ def _getatt_constructor(loader, node): def _tag_constructor(loader, tag_suffix, node): if tag_suffix not in CFN_FNS and tag_suffix not in CFN_TAGS: - raise ValueError("Bad tag: !{tag_suffix}. Supported tags are: " - "{supported_tags}".format( - tag_suffix=tag_suffix, - supported_tags=", ".join(sorted(CFN_TAGS + CFN_FNS)) - )) + raise ValueError( + "Bad tag: !{tag_suffix}. Supported tags are: " + "{supported_tags}".format( + tag_suffix=tag_suffix, + supported_tags=", ".join(sorted(CFN_TAGS + CFN_FNS)), + ) + ) if tag_suffix in CFN_FNS: tag_suffix = "Fn::{tag_suffix}".format(tag_suffix=tag_suffix) @@ -324,8 +461,8 @@ def _tag_constructor(loader, tag_suffix, node): data = {} yield data - if tag_suffix == 'Fn::GetAtt': - constructor = partial(_getatt_constructor, (loader, )) + if tag_suffix == "Fn::GetAtt": + constructor = partial(_getatt_constructor, (loader,)) elif isinstance(node, yaml.ScalarNode): constructor = loader.construct_scalar elif isinstance(node, yaml.SequenceNode): diff --git a/sceptre/cli/launch.py b/sceptre/cli/launch.py index 09ed462df..9b8514115 100644 --- a/sceptre/cli/launch.py +++ b/sceptre/cli/launch.py @@ -1,39 +1,199 @@ +import logging +from typing import List, Optional + import click +from click import Context +from colorama import Fore, Style +from sceptre.cli.helpers import catch_exceptions, confirmation, stack_status_exit_code +from sceptre.cli.prune import Pruner from sceptre.context import SceptreContext -from sceptre.cli.helpers import catch_exceptions -from sceptre.cli.helpers import confirmation -from sceptre.cli.helpers import stack_status_exit_code +from sceptre.exceptions import DependencyDoesNotExistError from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack + +logger = logging.getLogger(__name__) @click.command(name="launch", short_help="Launch a Stack or StackGroup.") @click.argument("path") +@click.option("-y", "--yes", is_flag=True, help="Assume yes to all questions.") +@click.option( + "-p", + "--prune", + is_flag=True, + help="If set, will delete all stacks in the command path marked as obsolete.", +) @click.option( - "-y", "--yes", is_flag=True, help="Assume yes to all questions." + "--disable-rollback/--enable-rollback", + default=None, + help="Disable or enable the cloudformation automatic rollback", ) @click.pass_context @catch_exceptions -def launch_command(ctx, path, yes): +def launch_command( + ctx: Context, path: str, yes: bool, prune: bool, disable_rollback: Optional[bool] +): """ - Launch a Stack or StackGroup for a given config PATH. - \f + Launch a Stack or StackGroup for a given config PATH. This command is intended as a catch-all + command that will apply any changes from Stack Configs indicated via the path. - :param path: The path to launch. Can be a Stack or StackGroup. - :type path: str - :param yes: A flag to answer 'yes' to all CLI questions. - :type yes: bool + \b + * Any Stacks that do not exist will be created + * Any stacks that already exist will be updated (if there are any changes) + * If any stacks are marked with "ignore: True", those stacks will neither be created nor updated + * If any stacks are marked with "obsolete: True", those stacks will neither be created nor updated. + * Furthermore, if the "-p"/"--prune" flag is used, these stacks will be deleted prior to any + other launch commands """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) + launcher = Launcher(context) + launcher.print_operations(prune) + if not yes: + launcher.confirm(prune) + + exit_code = launcher.launch(prune) + exit(exit_code) + + +class Launcher: + """Launcher is a utility to coordinate the flow of launching. + + :param context: The Sceptre context to use for launching + :param plan_factory: A callable with the signature of (SceptreContext) -> SceptrePlan + """ + + def __init__( + self, context: SceptreContext, plan_factory=SceptrePlan, pruner_factory=Pruner + ): + self._context = context + self._make_plan = plan_factory + self._make_pruner = pruner_factory + + self._plan = None + + def confirm(self, prune: bool): + self._confirm_launch(prune) + + def print_operations(self, prune: bool): + deploy_plan = self._create_deploy_plan() + stacks_to_skip = self._get_stacks_to_skip(deploy_plan, prune) + self._print_skips(stacks_to_skip) + if prune: + pruner = self._make_pruner(self._context, self._make_plan) + pruner.print_operations() + + def launch(self, prune: bool) -> int: + deploy_plan = self._create_deploy_plan() + stacks_to_skip = self._get_stacks_to_skip(deploy_plan, prune) + stacks_to_prune = self._get_stacks_to_prune(deploy_plan, prune) + + self._exclude_stacks_from_plan(deploy_plan, *stacks_to_skip, *stacks_to_prune) + self._validate_launch_for_missing_dependencies(deploy_plan, prune) + + code = 0 + if prune: + code = self._prune() + + code = code or self._deploy(deploy_plan) + return code + + def _create_deploy_plan(self) -> SceptrePlan: + if not self._plan: + plan = self._make_plan(self._context) + # The plan must be resolved so we can modify launch order and items before executing it + plan.resolve(plan.launch.__name__) + self._plan = plan + return self._plan + + def _get_stacks_to_skip(self, deploy_plan: SceptrePlan, prune: bool) -> List[Stack]: + return [ + stack + for stack in deploy_plan + if stack.ignore or (stack.obsolete and not prune) + ] + + def _get_stacks_to_prune( + self, deploy_plan: SceptrePlan, prune: bool + ) -> List[Stack]: + return [stack for stack in deploy_plan if prune and stack.obsolete] + + def _exclude_stacks_from_plan(self, deployment_plan: SceptrePlan, *stacks: Stack): + for stack in stacks: + deployment_plan.remove_stack_from_plan(stack) + + def _validate_launch_for_missing_dependencies( + self, deploy_plan: SceptrePlan, prune: bool + ): + validated_stacks = set() + skipped_dependencies = set() + + def validate_stack_dependencies(stack: Stack): + if stack in validated_stacks: + # In order to avoid unnecessary recursions on stacks already evaluated, we'll return + # early if we've already evaluated the stack without issue. + return + if prune and stack.obsolete: + raise DependencyDoesNotExistError( + f"Launch plan with --prune option depends on stack '{stack.name}' that is marked " + f"as obsolete. Only obsolete stacks can depend upon obsolete stacks when pruning." + ) + for dependency in stack.dependencies: + if dependency.ignore or dependency.obsolete: + skipped_dependencies.add(dependency) + if not self._context.ignore_dependencies: + validate_stack_dependencies(dependency) + validated_stacks.add(stack) + + for stack in deploy_plan: + validate_stack_dependencies(stack) + + message = ( + "WARNING: Launch plan depends on the following ignored and/or obsolete stacks.\n" + " Sceptre will attempt to continue with launch, but it may fail if any Stack Configs \n" + " require certain resources or outputs that don't currently exist." + ) + self._print_stacks_with_message(list(skipped_dependencies), message) + + def _print_skips(self, stacks_to_skip: List[Stack]): + skip_message = "During launch, the following stacks will be skipped, neither created nor updated:" + self._print_stacks_with_message(stacks_to_skip, skip_message) + + def _print_stacks_with_message(self, stacks: List[Stack], message: str): + if not len(stacks): + return + + message = f"* {message}\n" + for stack in stacks: + message += f"{Fore.YELLOW}{stack.name}{Style.RESET_ALL}\n" + + click.echo(message) + + def _print_deletions(self, stacks_to_prune: List[Stack]): + delete_message = "During launch, the following stacks will be will be deleted, if they exist:" + self._print_stacks_with_message(stacks_to_prune, delete_message) + + def _confirm_launch(self, prune: bool): + operation_name = "launch" + if prune: + operation_name += " --prune" + confirmation(operation_name, False, command_path=self._context.command_path) - plan = SceptrePlan(context) + def _prune(self) -> int: + pruner = self._make_pruner(self._context, self._make_plan) + exit_code = pruner.prune() + if exit_code != 0: + click.echo("Stack deletion failed, so could not proceed with launch.") + return exit_code - confirmation(plan.launch.__name__, yes, command_path=path) - responses = plan.launch() - exit(stack_status_exit_code(responses.values())) + def _deploy(self, deploy_plan: SceptrePlan) -> int: + result = deploy_plan.launch() + exit_code = stack_status_exit_code(result.values()) + return exit_code diff --git a/sceptre/cli/list.py b/sceptre/cli/list.py index 0714af476..6f11bd2cc 100644 --- a/sceptre/cli/list.py +++ b/sceptre/cli/list.py @@ -1,18 +1,17 @@ +import logging import click from sceptre.context import SceptreContext -from sceptre.cli.helpers import ( - catch_exceptions, - write -) +from sceptre.cli.helpers import catch_exceptions, write from sceptre.plan.plan import SceptrePlan +logger = logging.getLogger(__name__) + @click.group(name="list") def list_group(): """ Commands for listing attributes of stacks. - """ pass @@ -31,17 +30,17 @@ def list_resources(ctx, path): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), output_format=ctx.obj.get("output_format"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) responses = [ - response for response - in plan.describe_resources().values() if response + response for response in plan.describe_resources().values() if response ] write(responses, context.output_format) @@ -50,8 +49,10 @@ def list_resources(ctx, path): @list_group.command(name="outputs") @click.argument("path") @click.option( - "-e", "--export", type=click.Choice(["envvar"]), - help="Specify the export formatting." + "-e", + "--export", + type=click.Choice(["envvar"]), + help="Specify the export formatting.", ) @click.pass_context @catch_exceptions @@ -64,60 +65,92 @@ def list_outputs(ctx, path, export): :type path: str :param export: Specify the export formatting. :type export: str - """ + """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path", None), user_variables=ctx.obj.get("user_variables", {}), options=ctx.obj.get("options", {}), output_format=ctx.obj.get("output_format"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) - responses = [ - response for response - in plan.describe_outputs().values() if response - ] + responses = [response for response in plan.describe_outputs().values() if response] if export == "envvar": for response in responses: for stack in response.values(): for output in stack: - write("export SCEPTRE_{0}='{1}'".format( - output.get("OutputKey"), - output.get("OutputValue") - ), 'text') + write( + "export SCEPTRE_{0}='{1}'".format( + output.get("OutputKey"), output.get("OutputValue") + ), + "text", + ) else: write(responses, context.output_format) @list_group.command(name="change-sets") +@click.option("-U", "--url", is_flag=True, help="Instead write a URL.") @click.argument("path") @click.pass_context @catch_exceptions -def list_change_sets(ctx, path): +def list_change_sets(ctx, path, url): """ List change sets for stack. \f :param path: Path to execute the command on. :type path: str + :param url: Write out a console URL instead. + :type url: bool """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), output_format=ctx.obj.get("output_format"), options=ctx.obj.get("options"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) + responses = [ - response for response - in plan.list_change_sets().values() if response + response for response in plan.list_change_sets(url).values() if response ] for response in responses: write(response, context.output_format) + + +@list_group.command(name="stacks") +@click.argument("path") +@click.pass_context +@catch_exceptions +def list_stacks(ctx, path): + """ + List sceptre stack config attributes, + \f + + :param path: Path to execute the command on or path to stack group + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + output_format=ctx.obj.get("output_format"), + options=ctx.obj.get("options"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + ) + + plan = SceptrePlan(context) + + output = {f"{stack.name}.yaml": stack.external_name for stack in plan.graph} + output_format = "json" if context.output_format == "json" else "yaml" + write(output, output_format) diff --git a/sceptre/cli/new.py b/sceptre/cli/new.py index 8235e4cb4..66ec30869 100644 --- a/sceptre/cli/new.py +++ b/sceptre/cli/new.py @@ -18,8 +18,10 @@ def new_group(): pass -@new_group.command("group", short_help="Creates a new Stack Group directory in a project.") -@click.argument('stack_group') +@new_group.command( + "group", short_help="Creates a new Stack Group directory in a project." +) +@click.argument("stack_group") @catch_exceptions @click.pass_context def new_stack_group(ctx, stack_group): @@ -41,7 +43,7 @@ def new_stack_group(ctx, stack_group): @new_group.command("project", short_help="Creates a new project.") @catch_exceptions -@click.argument('project_name') +@click.argument("project_name") @click.pass_context def new_project(ctx, project_name): """ @@ -61,7 +63,7 @@ def new_project(ctx, project_name): # Check if stack_group folder already exists if e.errno == errno.EEXIST: raise ProjectAlreadyExistsError( - 'Folder \"{0}\" already exists.'.format(project_name) + 'Folder "{0}" already exists.'.format(project_name) ) else: raise @@ -72,7 +74,7 @@ def new_project(ctx, project_name): defaults = { "project_code": project_name, - "region": os.environ.get("AWS_DEFAULT_REGION", "") + "region": os.environ.get("AWS_DEFAULT_REGION", ""), } config_path = os.path.join(cwd, project_name, "config") @@ -92,7 +94,7 @@ def _create_new_stack_group(config_dir, new_path): """ # Create full path to stack_group folder_path = os.path.join(config_dir, new_path) - new_config_msg = 'Do you want initialise config.yaml?' + new_config_msg = "Do you want initialise config.yaml?" # Make folders for the stack_group try: @@ -100,8 +102,7 @@ def _create_new_stack_group(config_dir, new_path): except OSError as e: # Check if stack_group folder already exists if e.errno == errno.EEXIST: - new_config_msg =\ - 'StackGroup path exists. ' + new_config_msg + new_config_msg = "StackGroup path exists. " + new_config_msg else: raise @@ -158,9 +159,7 @@ def _create_config_file(config_dir, path, defaults={}): # Ask for new values for key, value in config.items(): - config[key] = click.prompt( - 'Please enter a {0}'.format(key), default=value - ) + config[key] = click.prompt("Please enter a {0}".format(key), default=value) # Remove values if parent config are the same config = {k: v for k, v in config.items() if parent_config.get(k) != v} @@ -168,9 +167,7 @@ def _create_config_file(config_dir, path, defaults={}): # Write config.yaml if config not empty filepath = os.path.join(path, "config.yaml") if config: - with open(filepath, 'w') as config_file: - yaml.safe_dump( - config, stream=config_file, default_flow_style=False - ) + with open(filepath, "w") as config_file: + yaml.safe_dump(config, stream=config_file, default_flow_style=False) else: click.echo("No config.yaml file needed - covered by parent config.") diff --git a/sceptre/cli/policy.py b/sceptre/cli/policy.py index 8ecb27d93..682167a29 100644 --- a/sceptre/cli/policy.py +++ b/sceptre/cli/policy.py @@ -9,8 +9,10 @@ @click.argument("path") @click.argument("policy-file", required=False) @click.option( - "-b", "--built-in", type=click.Choice(["deny-all", "allow-all"]), - help="Specify a built in stack policy." + "-b", + "--built-in", + type=click.Choice(["deny-all", "allow-all"]), + help="Specify a built in stack policy.", ) @click.pass_context @catch_exceptions @@ -28,16 +30,17 @@ def set_policy_command(ctx, path, policy_file, built_in): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) - if built_in == 'deny-all': + if built_in == "deny-all": plan.lock() - elif built_in == 'allow-all': + elif built_in == "allow-all": plan.unlock() else: plan.set_policy(policy_file) diff --git a/sceptre/cli/prune.py b/sceptre/cli/prune.py new file mode 100644 index 000000000..ad01bef88 --- /dev/null +++ b/sceptre/cli/prune.py @@ -0,0 +1,172 @@ +import click +from colorama import Fore, Style + +from sceptre.cli.helpers import catch_exceptions, stack_status_exit_code +from sceptre.context import SceptreContext +from sceptre.exceptions import CannotPruneStackError +from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack + +PATH_FOR_WHOLE_PROJECT = "." + + +@click.command(name="prune", short_help="Deletes all obsolete stacks in the project") +@click.option("-y", "--yes", is_flag=True, help="Assume yes to all questions.") +@click.argument("path", default=PATH_FOR_WHOLE_PROJECT) +@click.pass_context +@catch_exceptions +def prune_command(ctx, yes: bool, path): + """ + This command deletes all obsolete stacks in the project. Only obsolete stacks can be deleted + via prune; If any non-obsolete stacks depend on obsolete stacks, an error will be + raised and this command will fail. + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + full_scan=True, + ) + pruner = Pruner(context) + pruner.print_operations() + if not yes and pruner.prune_count > 0: + pruner.confirm() + + code = pruner.prune() + exit(code) + + +class Pruner: + """Pruner is a utility to coordinate the flow of deleting all stacks in the project that + are marked "obsolete". + + Note: The command_path on the passed context will be ignored; This command operates on the + entire project rather than on any particular command path. + + :param context: The Sceptre context to use for pruning + :param plan_factory: A callable with the signature of (SceptreContext) -> SceptrePlan + """ + + def __init__(self, context: SceptreContext, plan_factory=SceptrePlan): + self._context = context + self._make_plan = plan_factory + + self._plan = None + + def confirm(self): + self._confirm_prune() + + def print_operations(self): + plan = self._create_plan() + + if not self._plan_has_obsolete_stacks(plan): + self._print_no_obsolete_stacks() + return + + self._print_stacks_to_be_deleted(plan) + + @property + def prune_count(self) -> 0: + plan = self._create_plan() + if self._plan_has_obsolete_stacks(plan): + return len(list(plan)) + return 0 + + def prune(self) -> int: + plan = self._create_plan() + + if not self._plan_has_obsolete_stacks(plan): + return 0 + + if not self._context.ignore_dependencies: + self._validate_plan_for_dependencies_on_obsolete_stacks(plan) + + code = self._prune(plan) + return code + + def _create_plan(self): + if not self._plan: + context = self._context.clone() + context.full_scan = True + plan = self._make_plan(self._context) + if context.command_path == PATH_FOR_WHOLE_PROJECT: + stacks = plan.graph + else: + stacks = plan.command_stacks + + plan.command_stacks = {stack for stack in stacks if stack.obsolete} + self._resolve_plan(plan) + self._plan = plan + return self._plan + + def _plan_has_obsolete_stacks(self, plan: SceptrePlan): + return len(plan.command_stacks) > 0 + + def _print_no_obsolete_stacks(self): + click.echo( + "* There are no stacks marked obsolete, so there is nothing to prune." + ) + + def _resolve_plan(self, plan: SceptrePlan): + if len(plan.command_stacks) > 0: + # Prune is actually a particular kind of filtered deletion, so we use delete as the actual + # resolved command. + plan.resolve(plan.delete.__name__, reverse=True) + + def _validate_plan_for_dependencies_on_obsolete_stacks(self, plan: SceptrePlan): + def check_for_non_obsolete_dependencies(stack: Stack): + # If we've already established it as an obsolete stack to delete, we're good. + if stack in plan.command_stacks: + return + + # This check shouldn't be necessary, but we're just double-checking that it is indeed + # not obsolete. + if stack.obsolete: + return + + # Theoretically, we've already gathered up ALL obsolete stacks as command stacks. If + # we've hit this line, there's a problem. Now we just need to know what caused it. This + # block climbs down the dependency graph to see which obsolete stack caused this stack + # to be included in the plan. + for dependency in stack.dependencies: + if dependency.obsolete: + raise CannotPruneStackError( + f"Cannot prune obsolete stack {dependency.name} because stack {stack.name} " + f"depends on it but is not obsolete." + ) + + # If we get to this point, it means this stack isn't obsolete and none of its dependencies + # are either. That only happens it depends on another non-obsolete stack that depends on + # an obsolete stack. As a result, we're not going to blow up here and instead will + # continue iterating on the plan and will raise the error on a stack that directly + # depends on the obsolete stack. + return + + for stack in plan: + check_for_non_obsolete_dependencies(stack) + + def _print_stacks_to_be_deleted(self, plan: SceptrePlan): + delete_msg = ( + "* The following obsolete stacks will be deleted (if they exist on AWS):\n" + ) + + stacks_list = "" + for stack in plan: + # It's possible there could be stacks in the plan that aren't obsolete because those + # stacks depend on obsolete stacks. They won't pass validation, but that's not the + # point of this method. We'll just skip those here and fail validation later. + if not stack.obsolete: + continue + stacks_list += "{}{}{}\n".format(Fore.YELLOW, stack.name, Style.RESET_ALL) + + click.echo(delete_msg + stacks_list) + + def _confirm_prune(self): + click.confirm("Do you want to delete these stacks?", abort=True) + + def _prune(self, plan: SceptrePlan): + responses = plan.delete() + return stack_status_exit_code(responses.values()) diff --git a/sceptre/cli/status.py b/sceptre/cli/status.py index d3ffd1e4f..e9e972369 100644 --- a/sceptre/cli/status.py +++ b/sceptre/cli/status.py @@ -1,10 +1,7 @@ import click from sceptre.context import SceptreContext -from sceptre.cli.helpers import ( - catch_exceptions, - write -) +from sceptre.cli.helpers import catch_exceptions, write from sceptre.plan.plan import SceptrePlan @@ -23,16 +20,18 @@ def status_command(ctx, path): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), no_colour=ctx.obj.get("no_colour"), output_format=ctx.obj.get("output_format"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) responses = plan.get_status() - message = "\n".join("{}: {}".format(stack.name, status) - for stack, status in responses.items()) + message = "\n".join( + "{}: {}".format(stack.name, status) for stack, status in responses.items() + ) write(message, no_colour=context.no_colour) diff --git a/sceptre/cli/template.py b/sceptre/cli/template.py index c1f7a7453..05fc89709 100644 --- a/sceptre/cli/template.py +++ b/sceptre/cli/template.py @@ -1,19 +1,30 @@ -import click +import logging import webbrowser +import click +import deprecation + +from sceptre import __version__ +from sceptre.cli.helpers import catch_exceptions, write from sceptre.context import SceptreContext -from sceptre.cli.helpers import ( - catch_exceptions, - write -) +from sceptre.helpers import null_context from sceptre.plan.plan import SceptrePlan +from sceptre.resolvers.placeholders import use_resolver_placeholders_on_error + +logger = logging.getLogger(__name__) @click.command(name="validate", short_help="Validates the template.") +@click.option( + "-n", + "--no-placeholders", + is_flag=True, + help="If True, no placeholder values will be supplied for resolvers that cannot be resolved.", +) @click.argument("path") @click.pass_context @catch_exceptions -def validate_command(ctx, path): +def validate_command(ctx, no_placeholders, path): """ Validates the template used for stack in PATH. \f @@ -23,28 +34,41 @@ def validate_command(ctx, path): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), output_format=ctx.obj.get("output_format"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) - responses = plan.validate() + + execution_context = ( + null_context() if no_placeholders else use_resolver_placeholders_on_error() + ) + with execution_context: + responses = plan.validate() for stack, response in responses.items(): - if response['ResponseMetadata']['HTTPStatusCode'] == 200: - del response['ResponseMetadata'] + if response["ResponseMetadata"]["HTTPStatusCode"] == 200: + del response["ResponseMetadata"] click.echo("Template {} is valid. Template details:\n".format(stack.name)) write(response, context.output_format) +@deprecation.deprecated("4.0.0", "5.0.0", __version__, "Use dump template instead.") @click.command(name="generate", short_help="Prints the template.") +@click.option( + "-n", + "--no-placeholders", + is_flag=True, + help="If True, no placeholder values will be supplied for resolvers that cannot be resolved.", +) @click.argument("path") @click.pass_context @catch_exceptions -def generate_command(ctx, path): +def generate_command(ctx, no_placeholders, path): """ Prints the template used for stack in PATH. \f @@ -54,15 +78,22 @@ def generate_command(ctx, path): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), output_format=ctx.obj.get("output_format"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) - responses = plan.generate() + + execution_context = ( + null_context() if no_placeholders else use_resolver_placeholders_on_error() + ) + with execution_context: + responses = plan.generate() + output = [template for template in responses.values()] write(output, context.output_format) @@ -83,20 +114,55 @@ def estimate_cost_command(ctx, path): """ context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), output_format=ctx.obj.get("output_format"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) responses = plan.estimate_cost() for stack, response in responses.items(): - if response['ResponseMetadata']['HTTPStatusCode'] == 200: - del response['ResponseMetadata'] + if response["ResponseMetadata"]["HTTPStatusCode"] == 200: + del response["ResponseMetadata"] click.echo("View the estimated cost for {} at:".format(stack.name)) response = response["Url"] webbrowser.open(response, new=2) - write(response + "\n", 'text') + write(response + "\n", "text") + + +@click.command(name="fetch-remote-template", short_help="Prints the remote template.") +@click.argument("path") +@click.pass_context +@catch_exceptions +def fetch_remote_template_command(ctx, path): + """ + Prints the remote template used for stack in PATH. + \f + + :param path: Path to execute the command on. + :type path: str + """ + context = SceptreContext( + command_path=path, + command_params=ctx.params, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + output_format=ctx.obj.get("output_format"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + ) + + plan = SceptrePlan(context) + responses = plan.fetch_remote_template() + output = [] + for stack, template in responses.items(): + if template is None: + logger.warning(f"{stack.external_name} does not exist") + else: + output.append(template) + + write(output, context.output_format) diff --git a/sceptre/cli/update.py b/sceptre/cli/update.py index 56c9b242f..35d4239ae 100644 --- a/sceptre/cli/update.py +++ b/sceptre/cli/update.py @@ -2,6 +2,7 @@ import click +from typing import Optional from sceptre.context import SceptreContext from sceptre.cli.helpers import catch_exceptions, confirmation from sceptre.cli.helpers import write, stack_status_exit_code @@ -13,18 +14,20 @@ @click.command(name="update", short_help="Update a stack.") @click.argument("path") @click.option( - "-c", "--change-set", is_flag=True, - help="Create a change set before updating." + "-c", "--change-set", is_flag=True, help="Create a change set before updating." ) +@click.option("-v", "--verbose", is_flag=True, help="Display verbose output.") +@click.option("-y", "--yes", is_flag=True, help="Assume yes to all questions.") @click.option( - "-v", "--verbose", is_flag=True, help="Display verbose output." -) -@click.option( - "-y", "--yes", is_flag=True, help="Assume yes to all questions." + "--disable-rollback/--enable-rollback", + default=None, + help="Disable or enable the cloudformation automatic rollback", ) @click.pass_context @catch_exceptions -def update_command(ctx, path, change_set, verbose, yes): +def update_command( + ctx, path, change_set, verbose, yes, disable_rollback: Optional[bool] +): """ Updates a stack for a given config PATH. Or perform an update via change-set when the change-set flag is set. @@ -42,11 +45,12 @@ def update_command(ctx, path, change_set, verbose, yes): context = SceptreContext( command_path=path, + command_params=ctx.params, project_path=ctx.obj.get("project_path"), user_variables=ctx.obj.get("user_variables"), options=ctx.obj.get("options"), output_format=ctx.obj.get("output_format"), - ignore_dependencies=ctx.obj.get("ignore_dependencies") + ignore_dependencies=ctx.obj.get("ignore_dependencies"), ) plan = SceptrePlan(context) diff --git a/sceptre/config/__init__.py b/sceptre/config/__init__.py index 1656562d8..4f8bf5472 100644 --- a/sceptre/config/__init__.py +++ b/sceptre/config/__init__.py @@ -3,8 +3,8 @@ import logging -__author__ = 'Cloudreach' -__email__ = 'sceptre@cloudreach.com' +__author__ = "Cloudreach" +__email__ = "sceptre@cloudreach.com" # Set up logging to ``/dev/null`` like a library is supposed to. @@ -14,4 +14,4 @@ def emit(self, record): pass -logging.getLogger('sceptre').addHandler(NullHandler()) +logging.getLogger("sceptre").addHandler(NullHandler()) diff --git a/sceptre/config/graph.py b/sceptre/config/graph.py index 1f52b6a9f..34d6b79d8 100644 --- a/sceptre/config/graph.py +++ b/sceptre/config/graph.py @@ -8,8 +8,11 @@ """ import logging +from typing import List + import networkx as nx from sceptre.exceptions import CircularDependenciesError +from sceptre.stack import Stack class StackGraph(object): @@ -77,27 +80,23 @@ def _generate_graph(self, stacks): self._generate_edges(stack, stack.dependencies) self.graph.remove_edges_from(nx.selfloop_edges(self.graph)) - def _generate_edges(self, stack, dependencies): + def _generate_edges(self, stack: Stack, dependencies: List[Stack]): """ Adds edges to the graph based on a list of dependencies that are - generated from the inital stack config. Each of the paths + generated from the initial stack config. Each of the paths in the inital_dependency_paths list are a depency that the inital Stack config depends on. :param stack: A Sceptre Stack - :type stack: sceptre.stack.Stack :param dependencies: a collection of dependency paths - :type dependencies: list """ - self.logger.debug( - "Generate dependencies for stack {0}".format(stack) - ) - for dependency in dependencies: + self.logger.debug("Generate dependencies for stack {0}".format(stack)) + for dependency in set(dependencies): self.graph.add_edge(dependency, stack) if not nx.is_directed_acyclic_graph(self.graph): raise CircularDependenciesError( - "Dependency cycle detected: {} {}".format(stack, - dependency)) + f"Dependency cycle detected: {stack} {dependency}" + ) self.logger.debug(" Added dependency: {}".format(dependency)) if not dependencies: diff --git a/sceptre/config/reader.py b/sceptre/config/reader.py index 081c1a0d5..f618d87bd 100644 --- a/sceptre/config/reader.py +++ b/sceptre/config/reader.py @@ -8,13 +8,16 @@ """ import collections +import copy import datetime import fnmatch import logging -from os import environ, path, walk -from pkg_resources import iter_entry_points +import sys import yaml +from os import environ, path, walk +from typing import Set, Tuple +from pathlib import Path from jinja2 import Environment from jinja2 import StrictUndefined from jinja2 import FileSystemLoader @@ -23,6 +26,7 @@ from packaging.version import Version from sceptre import __version__ +from sceptre.exceptions import SceptreException from sceptre.exceptions import DependencyDoesNotExistError from sceptre.exceptions import InvalidConfigFileError from sceptre.exceptions import InvalidSceptreDirectoryError @@ -38,6 +42,9 @@ "dependencies": strategies.list_join, "hooks": strategies.child_wins, "iam_role": strategies.child_wins, + "sceptre_role": strategies.child_wins, + "iam_role_session_duration": strategies.child_wins, + "sceptre_role_session_duration": strategies.child_wins, "notifications": strategies.child_wins, "on_failure": strategies.child_wins, "parameters": strategies.child_wins, @@ -47,46 +54,52 @@ "region": strategies.child_wins, "required_version": strategies.child_wins, "role_arn": strategies.child_wins, + "cloudformation_service_role": strategies.child_wins, "sceptre_user_data": strategies.child_wins, "stack_name": strategies.child_wins, "stack_tags": strategies.child_wins, "stack_timeout": strategies.child_wins, "template_bucket_name": strategies.child_wins, "template_key_value": strategies.child_wins, - "template_path": strategies.child_wins + "template": strategies.child_wins, + "template_path": strategies.child_wins, + "ignore": strategies.child_wins, + "obsolete": strategies.child_wins, } STACK_GROUP_CONFIG_ATTRIBUTES = ConfigAttributes( - { - "project_code", - "region" - }, + {"project_code", "region"}, { "template_bucket_name", "template_key_prefix", - "required_version" - } + "required_version", + "j2_environment", + }, ) STACK_CONFIG_ATTRIBUTES = ConfigAttributes( + {}, { - "template_path" - }, - { + "template_path", + "template", "dependencies", "hooks", "iam_role", + "sceptre_role", + "iam_role_session_duration", + "sceptre_role_session_duration", "notifications", "on_failure", "parameters", "profile", "protect", "role_arn", + "cloudformation_service_role", "sceptre_user_data", "stack_name", "stack_tags", - "stack_timeout" - } + "stack_timeout", + }, ) INTERNAL_CONFIG_ATTRIBUTES = ConfigAttributes( @@ -94,8 +107,7 @@ "project_path", "stack_group_path", }, - { - } + {}, ) REQUIRED_KEYS = STACK_GROUP_CONFIG_ATTRIBUTES.required.union( @@ -129,6 +141,21 @@ def __init__(self, context): self.templating_vars = {"var": self.context.user_variables} + @staticmethod + def _iterate_entry_points(group): + """ + Helper to determine whether to use pkg_resources or importlib.metadata. + https://docs.python.org/3/library/importlib.metadata.html + """ + if sys.version_info < (3, 10): + from pkg_resources import iter_entry_points + + return iter_entry_points(group) + else: + from importlib.metadata import entry_points + + return entry_points(group=group) + def _add_yaml_constructors(self, entry_point_groups): """ Adds PyYAML constructor functions for all classes found registered at @@ -154,18 +181,19 @@ def constructor_factory(node_class): :returns: Class initialiser. :rtype: func """ - # This function signture is required by PyYAML + + # This function signature is required by PyYAML def class_constructor(loader, node): return node_class( - loader.construct_scalar(node) + loader.construct_object(self.resolve_node_tag(loader, node)) ) # pragma: no cover return class_constructor for group in entry_point_groups: - for entry_point in iter_entry_points(group): + for entry_point in self._iterate_entry_points(group): # Retrieve name and class from entry point - node_tag = u'!' + entry_point.name + node_tag = "!" + entry_point.name node_class = entry_point.load() # Add constructor to PyYAML loader @@ -174,10 +202,16 @@ def class_constructor(loader, node): ) self.logger.debug( "Added constructor for %s with node tag %s", - str(node_class), node_tag + str(node_class), + node_tag, ) - def construct_stacks(self): + def resolve_node_tag(self, loader, node): + node = copy.copy(node) + node.tag = loader.resolve(type(node), node.value, (True, False)) + return node + + def construct_stacks(self) -> Tuple[Set[Stack], Set[Stack]]: """ Traverses the files under the command path. For each file encountered, a Stack is constructed @@ -185,13 +219,13 @@ def construct_stacks(self): and a final set of Stacks is returned. :returns: A set of Stacks. - :rtype: set """ stack_map = {} command_stacks = set() - if self.context.ignore_dependencies: - root = self.context.full_command_path() - else: + + root = self.context.full_command_path() + + if self.context.full_scan: root = self.context.full_config_path() if path.isfile(root): @@ -199,37 +233,57 @@ def construct_stacks(self): else: todo = set() for directory_name, sub_directories, files in walk(root, followlinks=True): - for filename in fnmatch.filter(files, '*.yaml'): - if filename.startswith('config.'): + for filename in fnmatch.filter(files, "*.yaml"): + if filename.startswith("config."): continue todo.add(path.join(directory_name, filename)) stack_group_configs = {} + full_todo = todo.copy() + deps_todo = set() while todo: abs_path = todo.pop() - rel_path = path.relpath( - abs_path, start=self.context.full_config_path()) + rel_path = path.relpath(abs_path, start=self.context.full_config_path()) directory, filename = path.split(rel_path) if directory in stack_group_configs: stack_group_config = stack_group_configs[directory] else: - stack_group_config = stack_group_configs[directory] = \ - self.read(path.join(directory, self.context.config_file)) + stack_group_config = stack_group_configs[directory] = self.read( + path.join(directory, self.context.config_file) + ) stack = self._construct_stack(rel_path, stack_group_config) + for dep in stack.dependencies: + full_dep = str(Path(self.context.full_config_path(), dep)) + if not path.exists(full_dep): + raise DependencyDoesNotExistError( + "{stackname}: Dependency {dep} not found. " + "Please make sure that your dependencies stack_outputs " + "have their full path from `config` defined.".format( + stackname=stack.name, dep=dep + ) + ) + + if full_dep not in full_todo and full_dep not in deps_todo: + todo.add(full_dep) + deps_todo.add(full_dep) + stack_map[sceptreise_path(rel_path)] = stack - if abs_path.startswith(self.context.full_command_path()): + full_command_path = self.context.full_command_path() + if abs_path == full_command_path or abs_path.startswith( + full_command_path.rstrip(path.sep) + path.sep + ): command_stacks.add(stack) stacks = self.resolve_stacks(stack_map) return stacks, command_stacks - def resolve_stacks(self, stack_map): + def resolve_stacks(self, stack_map) -> Set[Stack]: """ Transforms map of Stacks into a set of Stacks, transforms dependencies from a list of Strings (stack names) to a list of Stacks. @@ -245,17 +299,25 @@ def resolve_stacks(self, stack_map): if not self.context.ignore_dependencies: for i, dep in enumerate(stack.dependencies): try: - stack.dependencies[i] = stack_map[sceptreise_path(dep)] + if not isinstance(dep, Stack): + # If the dependency was inherited from a stack group, it might already + # have been mapped and so doesn't need to be mapped again. + stack.dependencies[i] = stack_map[sceptreise_path(dep)] except KeyError: raise DependencyDoesNotExistError( "{stackname}: Dependency {dep} not found. " "Valid dependency names are: " "{stackkeys}. " "Please make sure that your dependencies stack_outputs " - "have their full path from `config` defined." - .format(stackname=stack.name, dep=dep, - stackkeys=", ".join(stack_map.keys()))) - + "have their full path from `config` defined.".format( + stackname=stack.name, + dep=dep, + stackkeys=", ".join(stack_map.keys()), + ) + ) + # We deduplicate the dependencies using a set here, since it's possible that a given + # dependency ends up in the list multiple times. + stack.dependencies = list(set(stack.dependencies)) else: stack.dependencies = [] stacks.add(stack) @@ -280,7 +342,7 @@ def read(self, rel_path, base_config=None): # Adding properties from class config = { "project_path": self.context.project_path, - "stack_group_path": directory_path + "stack_group_path": directory_path, } # Adding defaults from base config. @@ -288,20 +350,19 @@ def read(self, rel_path, base_config=None): config.update(base_config) # Check if file exists, but ignore config.yaml as can be inherited. - if not path.isfile(abs_path)\ - and not filename.endswith(self.context.config_file): + if not path.isfile(abs_path) and not filename.endswith( + self.context.config_file + ): raise ConfigFileNotFoundError( - "Config file \"{0}\" not found.".format(rel_path) + 'Config file "{0}" not found.'.format(rel_path) ) # Parse and read in the config files. this_config = self._recursive_read(directory_path, filename, config) if "dependencies" in config or "dependencies" in this_config: - this_config['dependencies'] = \ - CONFIG_MERGE_STRATEGIES['dependencies']( - this_config.get("dependencies"), - config.get("dependencies") + this_config["dependencies"] = CONFIG_MERGE_STRATEGIES["dependencies"]( + this_config.get("dependencies"), config.get("dependencies") ) config.update(this_config) @@ -310,7 +371,9 @@ def read(self, rel_path, base_config=None): self.logger.debug("Config: %s", config) return config - def _recursive_read(self, directory_path, filename, stack_group_config): + def _recursive_read( + self, directory_path: str, filename: str, stack_group_config: dict + ) -> dict: """ Traverses the directory_path, from top to bottom, reading in all relevant config files. If config attributes are encountered further @@ -318,13 +381,9 @@ def _recursive_read(self, directory_path, filename, stack_group_config): `CONFIG_MERGE_STRATEGIES` dict. :param directory_path: Relative directory path to config to read. - :type directory_path: str :param filename: File name for the config to read. - :type filename: dict :param stack_group_config: The loaded config file for the StackGroup - :type stack_group_config: dict :returns: Representation of inherited config. - :rtype: dict """ parent_directory = path.split(directory_path)[0] @@ -333,15 +392,19 @@ def _recursive_read(self, directory_path, filename, stack_group_config): config = {} if directory_path: - config = self._recursive_read(parent_directory, filename, stack_group_config) + config = self._recursive_read( + parent_directory, filename, stack_group_config + ) + + # Combine the stack_group_config with the nested config dict + config_group = stack_group_config.copy() + config_group.update(config) # Read config file and overwrite inherited properties - child_config = self._render(directory_path, filename, stack_group_config) or {} + child_config = self._render(directory_path, filename, config_group) or {} for config_key, strategy in CONFIG_MERGE_STRATEGIES.items(): - value = strategy( - config.get(config_key), child_config.get(config_key) - ) + value = strategy(config.get(config_key), child_config.get(config_key)) if value: child_config[config_key] = value @@ -367,23 +430,46 @@ def _render(self, directory_path, basename, stack_group_config): config = {} abs_directory_path = path.join(self.full_config_path, directory_path) if path.isfile(path.join(abs_directory_path, basename)): - jinja_env = Environment( - autoescape=select_autoescape( - disabled_extensions=('yaml',), + default_j2_environment_config = { + "autoescape": select_autoescape( + disabled_extensions=("yaml",), default=True, ), - loader=FileSystemLoader(abs_directory_path), - undefined=StrictUndefined + "loader": FileSystemLoader(abs_directory_path), + "undefined": StrictUndefined, + } + j2_environment_config = strategies.dict_merge( + default_j2_environment_config, + stack_group_config.get("j2_environment", {}), ) - template = jinja_env.get_template(basename) + j2_environment = Environment(**j2_environment_config) + + try: + template = j2_environment.get_template(basename) + except Exception as err: + raise SceptreException( + f"{Path(directory_path, basename).as_posix()} - {err}" + ) from err + self.templating_vars.update(stack_group_config) - rendered_template = template.render( - self.templating_vars, - command_path=self.context.command_path.split(path.sep), - environment_variable=environ - ) - config = yaml.safe_load(rendered_template) + try: + rendered_template = template.render( + self.templating_vars, + command_path=self.context.command_path.split(path.sep), + environment_variable=environ, + ) + except Exception as err: + raise SceptreException( + f"{Path(directory_path, basename).as_posix()} - {err}" + ) from err + + try: + config = yaml.safe_load(rendered_template) + except Exception as err: + raise ValueError( + "Error parsing {}:\n{}".format(abs_directory_path, err) + ) return config @@ -409,14 +495,12 @@ def _check_version(self, config): :raises: sceptre.exceptions.VersionIncompatibleException """ sceptre_version = __version__ - if 'required_version' in config: - required_version = config['required_version'] + if "required_version" in config: + required_version = config["required_version"] if Version(sceptre_version) not in SpecifierSet(required_version, True): raise VersionIncompatibleError( "Current sceptre version ({0}) does not meet version " - "requirements: {1}".format( - sceptre_version, required_version - ) + "requirements: {1}".format(sceptre_version, required_version) ) @staticmethod @@ -432,16 +516,19 @@ def _collect_s3_details(stack_name, config): :rtype: dict """ s3_details = None - if "template_bucket_name" in config: - template_key = "/".join([ - sceptreise_path(stack_name), "{time_stamp}.json".format( - time_stamp=datetime.datetime.utcnow().strftime( - "%Y-%m-%d-%H-%M-%S-%fZ" - ) - ) - ]) - - bucket_region = config.get("region", None) + # If the config explicitly sets the template_bucket_name to None, we don't want to enter + # this conditional block. + if config.get("template_bucket_name") is not None: + template_key = "/".join( + [ + sceptreise_path(stack_name), + "{time_stamp}.json".format( + time_stamp=datetime.datetime.utcnow().strftime( + "%Y-%m-%d-%H-%M-%S-%fZ" + ) + ), + ] + ) if "template_key_prefix" in config: prefix = config["template_key_prefix"] @@ -450,7 +537,6 @@ def _collect_s3_details(stack_name, config): s3_details = { "bucket_name": config["template_bucket_name"], "bucket_key": template_key, - "bucket_region": bucket_region } return s3_details @@ -485,23 +571,26 @@ def _construct_stack(self, rel_path, stack_group_config=None): ) ) - abs_template_path = path.join( - self.context.project_path, self.context.templates_path, - sceptreise_path(config["template_path"]) - ) + s3_details = self._collect_s3_details(stack_name, config) + # If disable/enable rollback was specified on the command line, use that. Otherwise, + # fall back to the stack config. + disable_rollback = self.context.command_params.get("disable_rollback") + if disable_rollback is None: + disable_rollback = config.get("disable_rollback", False) - s3_details = self._collect_s3_details( - stack_name, config - ) stack = Stack( name=stack_name, project_code=config["project_code"], - template_path=abs_template_path, + template_path=config.get("template_path"), + template_handler_config=config.get("template"), region=config["region"], template_bucket_name=config.get("template_bucket_name"), template_key_prefix=config.get("template_key_prefix"), required_version=config.get("required_version"), + sceptre_role=config.get("sceptre_role"), iam_role=config.get("iam_role"), + sceptre_role_session_duration=config.get("sceptre_role_session_duration"), + iam_role_session_duration=config.get("iam_role_session_duration"), profile=config.get("profile"), parameters=config.get("parameters", {}), sceptre_user_data=config.get("sceptre_user_data", {}), @@ -509,13 +598,17 @@ def _construct_stack(self, rel_path, stack_group_config=None): s3_details=s3_details, dependencies=config.get("dependencies", []), role_arn=config.get("role_arn"), + cloudformation_service_role=config.get("cloudformation_service_role"), protected=config.get("protect", False), tags=config.get("stack_tags", {}), external_name=config.get("stack_name"), notifications=config.get("notifications"), on_failure=config.get("on_failure"), + disable_rollback=disable_rollback, stack_timeout=config.get("stack_timeout", 0), - stack_group_config=parsed_stack_group_config + ignore=config.get("ignore", False), + obsolete=config.get("obsolete", False), + stack_group_config=parsed_stack_group_config, ) del self.templating_vars["stack_group_config"] @@ -529,9 +622,7 @@ def _parsed_stack_group_config(self, stack_group_config): """ parsed_config = { key: stack_group_config[key] - for key in - set(stack_group_config) - set(CONFIG_MERGE_STRATEGIES) + for key in set(stack_group_config) - set(CONFIG_MERGE_STRATEGIES) } - parsed_config.pop("project_path") parsed_config.pop("stack_group_path") return parsed_config diff --git a/sceptre/config/strategies.py b/sceptre/config/strategies.py index 125673de2..29b37d9ea 100644 --- a/sceptre/config/strategies.py +++ b/sceptre/config/strategies.py @@ -6,6 +6,7 @@ This module contains the implementations of the strategies used to merge config attributes. """ +from copy import deepcopy def list_join(a, b): @@ -20,17 +21,18 @@ def list_join(a, b): :rtype: list """ if a and not isinstance(a, list): - raise TypeError('{} is not a list'.format(a)) + raise TypeError("{} is not a list".format(a)) + if b and not isinstance(b, list): - raise TypeError('{} is not a list'.format(b)) + raise TypeError("{} is not a list".format(b)) if a is None: - return b + return deepcopy(b) if b is not None: - return a + b + return deepcopy(a + b) - return a + return deepcopy(a) def dict_merge(a, b): @@ -45,18 +47,17 @@ def dict_merge(a, b): :rtype: dict """ if a and not isinstance(a, dict): - raise TypeError('{} is not a dict'.format(a)) + raise TypeError("{} is not a dict".format(a)) if b and not isinstance(b, dict): - raise TypeError('{} is not a dict'.format(b)) + raise TypeError("{} is not a dict".format(b)) if a is None: - return b + return deepcopy(b) if b is not None: - a.update(b) - return a + return deepcopy({**a, **b}) - return a + return deepcopy(a) def child_wins(a, b): diff --git a/sceptre/connection_manager.py b/sceptre/connection_manager.py index 9b302552a..0e752c946 100644 --- a/sceptre/connection_manager.py +++ b/sceptre/connection_manager.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ sceptre.connection_manager @@ -9,16 +8,20 @@ import functools import logging +import os import random import threading import time -import boto3 +import warnings +from typing import Optional, Dict, Tuple, Any -from os import environ +import boto3 +import deprecation +from botocore.credentials import Credentials from botocore.exceptions import ClientError -from sceptre.helpers import mask_key from sceptre.exceptions import InvalidAWSCredentialsError, RetryLimitExceededError +from sceptre.helpers import mask_key, create_deprecated_alias_property def _retry_boto_call(func): @@ -63,9 +66,7 @@ def decorated(*args, **kwargs): else: raise raise RetryLimitExceededError( - "Exceeded request limit {0} times. Aborting.".format( - max_retries - ) + "Exceeded request limit {0} times. Aborting.".format(max_retries) ) return decorated @@ -77,71 +78,226 @@ class ConnectionManager(object): the various AWS services that Sceptre needs to interact with. :param profile: The AWS credentials profile that should be used. - :type profile: str - :param iam_role: The iam_role that should be assumed in the account. - :type iam_role: str + :param sceptre_role: The sceptre_role that should be assumed in the account. :param stack_name: The CloudFormation stack name for this connection. - :type stack_name: str :param region: The region to use. - :type region: str + :param sceptre_role_session_duration: The duration to assume the specified sceptre_role per session. """ + # STACK_DEFAULT is a sentinel value meaning "default to the stack's configuration". This is in + # contrast with passing None, which would mean "use no value". + STACK_DEFAULT = "[STACK DEFAULT]" + _session_lock = threading.Lock() _client_lock = threading.Lock() _boto_sessions = {} _clients = {} _stack_keys = {} - def __init__(self, region, profile=None, stack_name=None, iam_role=None): + iam_role = create_deprecated_alias_property( + "iam_role", "sceptre_role", "4.0.0", "5.0.0" + ) + sceptre_role_session_duration = 0 + iam_role_session_duration = create_deprecated_alias_property( + "iam_role_session_duration", "sceptre_role_session_duration", "4.0.0", "5.0.0" + ) + + def __init__( + self, + region: str, + profile: Optional[str] = None, + stack_name: Optional[str] = None, + sceptre_role: Optional[str] = None, + sceptre_role_session_duration: Optional[int] = None, + *, + session_class=boto3.Session, + get_envs_func=lambda: os.environ, + ): self.logger = logging.getLogger(__name__) self.region = region self.profile = profile self.stack_name = stack_name - self.iam_role = iam_role + self.sceptre_role = sceptre_role + self.sceptre_role_session_duration = sceptre_role_session_duration if stack_name: - self._stack_keys[stack_name] = (region, profile, iam_role) + self._stack_keys[stack_name] = (region, profile, sceptre_role) + + self._session_class = session_class + self._get_envs = get_envs_func def __repr__(self): return ( "sceptre.connection_manager.ConnectionManager(region='{0}', " - "profile='{1}', stack_name='{2}', iam_role='{3}')".format( - self.region, self.profile, self.stack_name, self.iam_role + "profile='{1}', stack_name='{2}', sceptre_role='{3}', sceptre_role_session_duration='{4}')".format( + self.region, + self.profile, + self.stack_name, + self.sceptre_role, + self.sceptre_role_session_duration, ) ) - def _get_session(self, profile, region, iam_role): + def get_session( + self, + profile: Optional[str] = STACK_DEFAULT, + region: Optional[str] = STACK_DEFAULT, + sceptre_role: Optional[str] = STACK_DEFAULT, + *, + iam_role: Optional[str] = STACK_DEFAULT, + ) -> boto3.Session: """ - Returns a boto session in the target account. - - If a ``profile`` is specified in ConnectionManager's initialiser, - then the profile is used to generate temporary credentials to create - the Boto session. If ``profile`` is not specified then the default - profile is assumed to create the boto session. + Returns a boto3 session for the targeted profile, region, and sceptre_role. + + For each of profile, region, and sceptre_role, these values will default to the ConnectionManager's + configured default values (which correspond to the Stack's configuration). These values can + be overridden, however, by passing them explicitly. + + :param profile: The name of the AWS Profile as configured in the local environment. Passing + None will result in no profile being specified. Defaults to the ConnectionManager's + configured profile (if there is one). + :param region: The AWS Region the session should be configured with. Defaults to the + ConnectionManager's configured region. + :param sceptre_role: The IAM role ARN that is assumed using STS to create the session. Passing + None will result in no IAM role being assumed. Defaults to the ConnectionManager's + configured sceptre_role (if there is one). + :param iam_role: An alias for sceptre_role; Deprecated in v4.0.0 and will be removed in + v5.0.0. :returns: The Boto3 session. - :rtype: boto3.session.Session :raises: botocore.exceptions.ClientError """ + profile, region, sceptre_role = self._determine_session_args( + profile, region, sceptre_role, iam_role + ) + + return self._get_session(profile, region, sceptre_role) + + def _determine_session_args( + self, profile: str, region: str, sceptre_role: str, iam_role: str + ) -> Tuple[str, str, str]: + profile = self.profile if profile == self.STACK_DEFAULT else profile + region = self.region if region == self.STACK_DEFAULT else region + sceptre_role = self._coalesce_sceptre_role(iam_role, sceptre_role) + sceptre_role = ( + self.sceptre_role if sceptre_role == self.STACK_DEFAULT else sceptre_role + ) + # For historical reasons, if all three values are "None", that means we default to the + # Stack's configuration. + if (profile, region, sceptre_role) == (None, None, None): + profile, region, sceptre_role = self.profile, self.region, self.sceptre_role + + return profile, region, sceptre_role + + def _emit_iam_role_deprecation_warning(self): + warnings.warn( + deprecation.DeprecatedWarning( + "The iam_role parameter", "4.0.0", "5.0.0", "Use sceptre_role instead" + ), + DeprecationWarning, + stacklevel=3, + ) + + def create_session_environment_variables( + self, + profile: Optional[str] = STACK_DEFAULT, + region: Optional[str] = STACK_DEFAULT, + sceptre_role: Optional[str] = STACK_DEFAULT, + include_system_envs: bool = True, + ) -> Dict[str, str]: + """Creates the standard AWS environment variables that would need to be passed to a + subprocess in a hook, resolver, or template handler and allow that subprocess to work with + the currently configured session. + + The environment variables returned by this method should be everything needed for + subprocesses to properly interact with AWS using the ConnectionManager's configurations for + profile, sceptre_role, and region. By default, they include the other process environment + variables, such as PATH and any others. If you do not want the other environment variables, + you can toggle these off via include_system_envs=False. + + | Notes on including system envs: + | * The AWS_DEFAULT_REGION, AWS_REGION, AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY + | environment variables (if they are set in the Sceptre process) will be overwritten in + | the returned dict with the correct values from the newly created Session. + | * If the AWS_SESSION_TOKEN environment variable is currently set for the process, this + | will be overwritten with the new session's token (if there is one) or removed from the + | returned environment variables dict (if the new session doesn't have a token). + + :param profile: The name of the AWS Profile as configured in the local environment. Passing + None will result in no profile being specified. Defaults to the ConnectionManager's + configured profile (if there is one). + :param region: The AWS Region the session should be configured with. Defaults to the + ConnectionManager's configured region. + :param sceptre_role: The IAM role ARN that is assumed using STS to create the session. Passing + None will result in no IAM role being assumed. Defaults to the ConnectionManager's + configured sceptre_role (if there is one). + :param include_system_envs: If True, will return a dict with all the system environment + variables included. This is useful for creating a complete dict of environment variables + to pass to a subprocess. If set to False, this method will ONLY return the relevant AWS + environment variables. Defaults to True. + + :returns: A dict of environment variables with the appropriate credentials available for use. + """ + session = self.get_session(profile, region, sceptre_role) + # Set aws environment variables specific to whatever AWS configuration has been set on the + # stack's connection manager. + credentials: Credentials = session.get_credentials() + envs = dict(**self._get_envs()) if include_system_envs else {} + + if include_system_envs: + # We don't want a profile specified, since that could interfere with the credentials we're + # about to set. Even if we're using a profile, the credentials will already reflect that + # profile's configurations. + envs.pop("AWS_PROFILE", None) + + envs.update( + AWS_ACCESS_KEY_ID=credentials.access_key, + AWS_SECRET_ACCESS_KEY=credentials.secret_key, + # Most AWS SDKs use AWS_DEFAULT_REGION for the region; some use AWS_REGION + AWS_DEFAULT_REGION=session.region_name, + AWS_REGION=session.region_name, + ) + + if credentials.token: + envs["AWS_SESSION_TOKEN"] = credentials.token + # There might not be a session token, so if there isn't one, make sure it doesn't exist in + # the envs being passed to the subprocess + elif include_system_envs: + envs.pop("AWS_SESSION_TOKEN", None) + + return envs + + def _get_session( + self, + profile: Optional[str], + region: Optional[str], + sceptre_role: Optional[str], + *, + iam_role: Optional[str] = None, + ) -> boto3.Session: + if iam_role is not None: + self._emit_iam_role_deprecation_warning() + sceptre_role = iam_role + with self._session_lock: self.logger.debug("Getting Boto3 session") - key = (region, profile, iam_role) + key = (region, profile, sceptre_role) if self._boto_sessions.get(key) is None: self.logger.debug("No Boto3 session found, creating one...") self.logger.debug("Using cli credentials...") - + environ = self._get_envs() # Credentials from env take priority over profile config = { "profile_name": profile, "region_name": region, "aws_access_key_id": environ.get("AWS_ACCESS_KEY_ID"), "aws_secret_access_key": environ.get("AWS_SECRET_ACCESS_KEY"), - "aws_session_token": environ.get("AWS_SESSION_TOKEN") + "aws_session_token": environ.get("AWS_SESSION_TOKEN"), } - session = boto3.session.Session(**config) + session = self._session_class(**config) self._boto_sessions[key] = session if session.get_credentials() is None: @@ -151,27 +307,32 @@ def _get_session(self, profile, region, iam_role): ) ) - if iam_role: + if sceptre_role: sts_client = session.client("sts") - sts_response = sts_client.assume_role( - RoleArn=iam_role, - RoleSessionName="{0}-session".format( - iam_role.split("/")[-1] - ) - ) + # maximum session name length is 64 chars. 56 + "-session" = 64 + session_name = f'{sceptre_role.split("/")[-1][:56]}-session' + assume_role_kwargs = { + "RoleArn": sceptre_role, + "RoleSessionName": session_name, + } + if self.sceptre_role_session_duration: + assume_role_kwargs[ + "DurationSeconds" + ] = self.sceptre_role_session_duration + sts_response = sts_client.assume_role(**assume_role_kwargs) credentials = sts_response["Credentials"] - session = boto3.session.Session( + session = self._session_class( aws_access_key_id=credentials["AccessKeyId"], aws_secret_access_key=credentials["SecretAccessKey"], aws_session_token=credentials["SessionToken"], - region_name=region + region_name=region, ) if session.get_credentials() is None: raise InvalidAWSCredentialsError( "Session credentials were not found. Role: {0}. Region: {1}.".format( - iam_role, region + sceptre_role, region ) ) @@ -181,21 +342,19 @@ def _get_session(self, profile, region, iam_role): "Using credential set from %s: %s", session.get_credentials().method, { - "AccessKeyId": mask_key( - session.get_credentials().access_key - ), + "AccessKeyId": mask_key(session.get_credentials().access_key), "SecretAccessKey": mask_key( session.get_credentials().secret_key ), - "Region": session.region_name - } + "Region": session.region_name, + }, ) self.logger.debug("Boto3 session created") return self._boto_sessions[key] - def _get_client(self, service, region, profile, stack_name, iam_role): + def _get_client(self, service, region, profile, stack_name, sceptre_role): """ Returns the Boto3 client associated with . @@ -208,46 +367,110 @@ def _get_client(self, service, region, profile, stack_name, iam_role): :rtype: boto3.client.Client """ with self._client_lock: - key = (service, region, profile, stack_name, iam_role) + key = (service, region, profile, stack_name, sceptre_role) if self._clients.get(key) is None: - self.logger.debug( - "No %s client found, creating one...", service - ) + self.logger.debug("No %s client found, creating one...", service) self._clients[key] = self._get_session( - profile, region, iam_role + profile, region, sceptre_role ).client(service) return self._clients[key] @_retry_boto_call def call( - self, service, command, kwargs=None, profile=None, region=None, - stack_name=None, iam_role=None + self, + service: str, + command: str, + kwargs: Dict[str, Any] = None, + profile: Optional[str] = STACK_DEFAULT, + region: Optional[str] = STACK_DEFAULT, + stack_name: Optional[str] = None, + sceptre_role: Optional[str] = STACK_DEFAULT, + *, + iam_role: Optional[str] = STACK_DEFAULT, ): """ Makes a thread-safe Boto3 client call. Equivalent to ``boto3.client().(**kwargs)``. + | Note regarding the profile, region, and sceptre_role parameters: + | We will interpret each parameter individually this way: + | * If the value passed is the STACK_DEFAULT constant, we'll assume it to mean we ought + | to use the target stack's value of that parameter. + | * If the value passed is None, we will interpret that as an explicit request to nullify + | the target stack's setting. Note: While this is valid for profile and sceptre_role, + | it will likely blow up if doing this for region, since AWS almost always requires that. + | * Otherwise, any value that has been specified will override the target stack's + | configuration, regardless of what has been passed for other parameters. + | * In the case that `None` has been specified for all parameters, that will be + | interpreted as using the target stack's values for all three, falling back to the + | current stack. + :param service: The Boto3 service to return a client for. - :type service: str :param command: The Boto3 command to call. - :type command: str :param kwargs: The keyword arguments to supply to . - :type kwargs: dict + :param profile: The profile to use when invoking the command; Defaults to the stack's configuration + :param region: The region to use when invoking the command; Default's to the stack's configuration + :param stack_name: The name of the stack whose configuration to use. Defaults to the current stack + :param sceptre_role: The IAM Role ARN to assume in order to invoke the command; Defaults to + the stack's configuration. + :param iam_role: DEPRECATED. Use sceptre_role instead. :returns: The response from the Boto3 call. - :rtype: dict """ - if region is None and profile is None and iam_role is None: - if stack_name and stack_name in self._stack_keys: - region, profile, iam_role = self._stack_keys[stack_name] + # If stack_name has been specified and we've already cached the region/profile/role + # configured for that stack, the "defaults" we'll use will be those of that stack rather then + # the defaults for the current ConnectionManager instance. + # + # stack_name is not used often and only really makes sense when we are acting inside Stack A + # but needing to interact with Stack B using Stack B's configurations. This is mostly only + # done when we're getting Stack B's outputs to resolve for Stack A's configuration. + if stack_name and stack_name in self._stack_keys: + stack_region, stack_profile, stack_sceptre_role = self._stack_keys[ + stack_name + ] + sceptre_role = self._coalesce_sceptre_role(iam_role, sceptre_role) + # For historical/legacy purposes, if `None` is explicitly passed for all three parameters, + # this will be interpreted to mean we're going to use the profile/region/role configuration + # of the stack name. This could potentially interfere with an explicit attempt to nullify + # a setting; However, in that case, we'd need to be setting the region to None... which + # is unlikely, since that is a required stack configuration. This is the way this + # function has always operated, so to change this behavior could break or cause + # unexpected behavior elsewhere. + if (region, profile, sceptre_role) == (None, None, None): + region, profile, sceptre_role = ( + stack_region, + stack_profile, + stack_sceptre_role, + ) + # In every other circumstance, we will interpret each parameter individually according + # to the way described in the docstring. else: - region = self.region - profile = self.profile - iam_role = self.iam_role + region = stack_region if region == self.STACK_DEFAULT else region + profile = stack_profile if profile == self.STACK_DEFAULT else profile + sceptre_role = ( + stack_sceptre_role + if sceptre_role == self.STACK_DEFAULT + else sceptre_role + ) + # In most cases, we won't be targeting another stack's configurations. Instead, we'll want + # to be using the configurations of the CURRENT stack. + else: + profile, region, sceptre_role = self._determine_session_args( + profile, region, sceptre_role, iam_role + ) if kwargs is None: # pragma: no cover kwargs = {} - client = self._get_client(service, region, profile, stack_name, iam_role) + client = self._get_client(service, region, profile, stack_name, sceptre_role) return getattr(client, command)(**kwargs) + + def _coalesce_sceptre_role(self, iam_role: str, sceptre_role: str) -> str: + """Evaluates the iam_role and sceptre_role parameters as passed to determine which value to + use. + """ + if sceptre_role == self.STACK_DEFAULT and iam_role != self.STACK_DEFAULT: + self._emit_iam_role_deprecation_warning() + sceptre_role = iam_role + return sceptre_role diff --git a/sceptre/context.py b/sceptre/context.py index c44209a87..c958c97de 100644 --- a/sceptre/context.py +++ b/sceptre/context.py @@ -6,7 +6,7 @@ This module implements the SceptreContext class which holds details about the paths used in a Sceptre project. """ - +from copy import deepcopy from os import path from sceptre.helpers import normalise_path @@ -40,11 +40,24 @@ class SceptreContext(object): :param no_colour: Specify whether colouring should be used in the CLI\ output :type no_colour: bool + + :param full_scan: Specify whether folder scan the config files\ + True for scan all the config files and False for scan only in the command path + :type full_scan: bool """ - def __init__(self, project_path, command_path, - user_variables=None, options=None, output_format=None, - no_colour=False, ignore_dependencies=False): + def __init__( + self, + project_path, + command_path, + command_params=None, + user_variables=None, + options=None, + output_format=None, + no_colour=False, + ignore_dependencies=False, + full_scan=False, + ): # project_path: absolute path to the base sceptre project folder # e.g. absolute_path/to/sceptre_directory self.project_path = normalise_path(project_path) @@ -59,6 +72,9 @@ def __init__(self, project_path, command_path, self.normal_command_path = normalise_path(command_path) + # the sceptre command parameters (e.g. sceptre launch ) + self.command_params = command_params or {} + # config_file: stack group config. User definable later in v2 # e.g. {project_path/config/command_path}/config_file self.config_file = "config.yaml" @@ -68,12 +84,14 @@ def __init__(self, project_path, command_path, self.templates_path = "templates" self.user_variables = user_variables if user_variables else {} - self.user_variables = user_variables\ - if user_variables is not None else {} + self.user_variables = user_variables if user_variables is not None else {} self.options = options if options else {} self.output_format = output_format if output_format else "" self.no_colour = no_colour if no_colour is True else False - self.ignore_dependencies = ignore_dependencies if ignore_dependencies is True else False + self.ignore_dependencies = ( + ignore_dependencies if ignore_dependencies is True else False + ) + self.full_scan = full_scan if full_scan is True else False def full_config_path(self): """ @@ -92,8 +110,7 @@ def full_command_path(self): :returns: The absolute path to the path that will be executed :rtype: str """ - return path.join(self.project_path, self.config_path, - self.command_path) + return path.join(self.project_path, self.config_path, self.command_path) def full_templates_path(self): """ @@ -112,9 +129,11 @@ def command_path_is_stack(self): :rtype: bool """ return path.isfile( - path.join( - self.project_path, - self.config_path, - self.command_path - ) + path.join(self.project_path, self.config_path, self.command_path) ) + + def clone(self) -> "SceptreContext": + """Creates a new, deep clone of the context with all the same values.""" + new = type(self).__new__(type(self)) + new.__dict__.update(deepcopy(self.__dict__)) + return new diff --git a/sceptre/diffing/__init__.py b/sceptre/diffing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sceptre/diffing/diff_writer.py b/sceptre/diffing/diff_writer.py new file mode 100644 index 000000000..b79c259ab --- /dev/null +++ b/sceptre/diffing/diff_writer.py @@ -0,0 +1,224 @@ +import datetime +import json +import re +from abc import abstractmethod +from typing import TextIO, Generic, List +from colorama import Fore + +import cfn_flip +import yaml +from deepdiff import DeepDiff +from deepdiff.serialization import json_convertor_default + +from sceptre.diffing.stack_differ import StackConfiguration, StackDiff, DiffType + +deepdiff_json_defaults = { + datetime.date: lambda x: x.isoformat(), + StackConfiguration: lambda x: dict(x._asdict()), +} + + +class DiffWriter(Generic[DiffType]): + """A component responsible for taking a StackDiff and writing it in a way that is useful and + readable. This is an abstract base class, so the abstract methods need to be implemented to + create a DiffWriter for a given DiffType. + """ + + STAR_BAR = "*" * 80 + LINE_BAR = "-" * 80 + + def __init__( + self, stack_diff: StackDiff, output_stream: TextIO, output_format: str + ): + """Initializes the DiffWriter + + :param stack_diff: The diff this writer will be outputting + :param output_stream: The stream this writer should output to; Generally, this will be + stdout + :param output_format: Output format specified for the base Sceptre cli command; This should + be one of "yaml", "json", or "text" + """ + self.stack_name = stack_diff.stack_name + self.stack_diff = stack_diff + + self.template_diff = stack_diff.template_diff + self.config_diff = stack_diff.config_diff + self.is_deployed = stack_diff.is_deployed + + self.output_stream = output_stream + self.output_format = output_format + + @property + def has_difference(self) -> bool: + return self.has_config_difference or self.has_template_difference + + def write(self): + """Writes the diff to the output stream.""" + self._output(self.STAR_BAR) + if not self.has_difference: + self._output(f"No difference to deployed stack {self.stack_name}") + return + + self._output(f"--> Difference detected for stack {self.stack_name}!") + + if not self.is_deployed: + self._write_new_stack_details() + return + + self._output(self.LINE_BAR) + self._write_config_difference() + self._output(self.LINE_BAR) + self._write_template_difference() + + def _write_new_stack_details(self): + stack_config_text = self._dump_stack_config(self.stack_diff.generated_config) + self._output( + "This stack is not deployed yet!", + self.LINE_BAR, + "New Config:", + "", + stack_config_text, + self.LINE_BAR, + "New Template:", + "", + self.stack_diff.generated_template, + ) + return + + def _output(self, *lines: str): + lines_with_breaks = [f"{line}\n" for line in lines] + self.output_stream.writelines(lines_with_breaks) + + def _dump_stack_config(self, stack_config: StackConfiguration) -> str: + stack_config_dict = dict(stack_config._asdict()) + dumped = self._dump_dict(stack_config_dict) + return dumped + + def _dump_dict(self, dict_to_dump: dict) -> str: + if self.output_format == "json": + # There's not really a viable way to dump a template as "text" -> YAML is very readable + dumper = cfn_flip.dump_json + else: + dumper = cfn_flip.dump_yaml + + dumped = dumper(dict_to_dump) + return dumped + + def _write_config_difference(self): + if not self.has_config_difference: + self._output("No stack config difference") + return + + diff_text = self.dump_diff(self.config_diff) + self._output(f"Config difference for {self.stack_name}:", "", diff_text) + + def _write_template_difference(self): + if not self.has_template_difference: + self._output("No template difference") + return + + diff_text = self.dump_diff(self.template_diff) + self._output(f"Template difference for {self.stack_name}:", "", diff_text) + + @abstractmethod + def dump_diff(self, diff: DiffType) -> str: + """ "Implement this method to write the DiffType to string""" + + @property + @abstractmethod + def has_config_difference(self) -> bool: + """Implement this to indicate whether or not there is a config difference""" + + @property + @abstractmethod + def has_template_difference(self) -> bool: + """Implement this to indicate whether or not there is a template difference""" + + +class DeepDiffWriter(DiffWriter[DeepDiff]): + """A DiffWriter for StackDiffs where the DiffType is a DeepDiff object.""" + + @property + def has_config_difference(self) -> bool: + return len(self.config_diff) > 0 + + @property + def has_template_difference(self) -> bool: + return len(self.template_diff) > 0 + + def dump_diff(self, diff: DeepDiff) -> str: + as_diff_dict = diff.to_dict() + if self.output_format == "json": + return json.dumps( + as_diff_dict, + indent=4, + default=json_convertor_default(default_mapping=deepdiff_json_defaults), + ) + + compatible = self._make_strings_block_compatible(as_diff_dict) + return yaml.dump(compatible, indent=4) + + def _make_strings_block_compatible(self, obj): + """A recursive method that strips out extraneous spaces that precede line breaks. + + PyYaml disallows block styling for multiline strings if any of the lines has a space followed + by a line break. + + DeepDiff will actually provide a difflib-style diff for multiline strings when there has + been a value changed from one multiline string to another multiline string. However, when + it produces that diff, every line ends with at least one space. This keeps it from being + formatted as a block (the most useful way to display it) by PyYaml. Therefore, this function + recurses into the deepdiff-generated data structure and strips all strings of those extraneous + spaces that precede line breaks. + + :param obj: The DeepDiff generated diff dict (or some value this method has recursed into + from that dict). + :return: The object, stripped of extraneous spaces that precede line breaks. + """ + if isinstance(obj, dict): + return { + key: self._make_strings_block_compatible(value) + for key, value in obj.items() + } + elif isinstance(obj, list): + return [self._make_strings_block_compatible(item) for item in obj] + elif isinstance(obj, str): + return re.sub("[ ]*\n", "\n", obj) + else: + return obj + + +class DiffLibWriter(DiffWriter[List[str]]): + """A DiffWriter for StackDiffs where the DiffType is a a list of strings.""" + + @property + def has_config_difference(self) -> bool: + return len(self.config_diff) > 0 + + @property + def has_template_difference(self) -> bool: + return len(self.template_diff) > 0 + + def dump_diff(self, diff: List[str]) -> str: + # Difflib doesn't care about the output format since it only outputs strings. We would have + # accounted for the output format in the differ itself rather than here. + return "\n".join(diff) + + +class ColouredDiffLibWriter(DiffLibWriter): + """A DiffWriter for StackDiffs where the DiffType is a a list of strings with coloured diffs.""" + + def _colour_diff(self, diff: List[str]): + for line in diff: + if line.startswith("+"): + yield Fore.GREEN + line + Fore.RESET + elif line.startswith("-"): + yield Fore.RED + line + Fore.RESET + elif line.startswith("^"): + yield Fore.BLUE + line + Fore.RESET + else: + yield line + + def dump_diff(self, diff: List[str]) -> str: + coloured_diff = self._colour_diff(diff) + return super().dump_diff(coloured_diff) diff --git a/sceptre/diffing/stack_differ.py b/sceptre/diffing/stack_differ.py new file mode 100644 index 000000000..eeb942e83 --- /dev/null +++ b/sceptre/diffing/stack_differ.py @@ -0,0 +1,474 @@ +import difflib +import logging +from abc import abstractmethod +from typing import ( + NamedTuple, + Dict, + List, + Optional, + Callable, + Tuple, + Generic, + TypeVar, + Union, +) + +import cfn_flip +import deepdiff +import yaml +from cfn_tools import ODict +from yaml import Dumper + +from sceptre.plan.actions import StackActions +from sceptre.stack import Stack + +DiffType = TypeVar("DiffType") + +logger = logging.getLogger(__name__) + + +class StackConfiguration(NamedTuple): + """A data container to represent the comparable parts of a Stack.""" + + stack_name: str + parameters: Dict[str, Union[str, List[str]]] + stack_tags: Dict[str, str] + notifications: List[str] + cloudformation_service_role: Optional[str] + + +class StackDiff(NamedTuple): + """A data container to represent the full difference between a deployed stack and the stack as + it exists locally within Sceptre. + """ + + stack_name: str + template_diff: DiffType + config_diff: DiffType + is_deployed: bool + generated_config: StackConfiguration + generated_template: str + + +def repr_str(dumper: Dumper, data: str) -> str: + """A YAML Representer that handles strings, breaking multi-line strings into something a lot + more readable in the yaml output. This is useful for representing long, multiline strings in + templates or in stack parameters. + + :param dumper: The Dumper that is being used to serialize this object + :param data: The string to serialize + :return: The represented string + """ + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_str(data) + + +def repr_odict(dumper: Dumper, data: ODict) -> str: + """A YAML Representer for cfn-flip's ODict objects. + + ODicts are a variation on OrderedDicts for that library. Since the Diff command makes extensive + use of ODicts, they can end up in diff output and the PyYaml library doesn't otherwise calls the + dicts like !!ODict, which looks weird. We can just treat them like normal dicts when we serialize + them, though. + + :param dumper: The Dumper that is being used to serialize this object + :param data: The ODict object to serialize + :return: The serialized ODict + """ + return dumper.represent_dict(data) + + +yaml.add_representer(str, repr_str) +yaml.add_representer(ODict, repr_odict) + + +class StackDiffer(Generic[DiffType]): + """A utility for producing a StackDiff that indicates the full difference between a given stack + as it is currently DEPLOYED on CloudFormation and the stack as it exists in the local Sceptre + configurations. + + This utility compares both the stack configuration (specifically those attributes that CAN be + compared) as well as the stack template. + + As an abstract base class, the two comparison methods need to be implemented so that the + StackDiff can be generated. + """ + + STACK_STATUSES_INDICATING_NOT_DEPLOYED = [ + "CREATE_FAILED", + "ROLLBACK_COMPLETE", + "DELETE_COMPLETE", + ] + + NO_ECHO_REPLACEMENT = "***HIDDEN***" + + def __init__(self, show_no_echo=False): + """Initializes the StackDiffer. + + :param show_no_echo: If True, local parameters passed that the template says are NoEcho + parameters will be displayed; Otherwise, they will be masked in the diff. + """ + self.show_no_echo = show_no_echo + + def diff(self, stack_actions: StackActions) -> StackDiff: + """Produces a StackDiff between the currently deployed stack (if it exists) and the stack + as it exists locally in Sceptre. + + :param stack_actions: The StackActions object to use for generating and fetching templates + as well as providing other details about the stack. + :return: The StackDiff that expresses the difference. + """ + generated_config = self._create_generated_config(stack_actions.stack) + deployed_config = self._create_deployed_stack_config(stack_actions) + is_stack_deployed = bool(deployed_config) + + generated_template = self._generate_template(stack_actions) + deployed_template = self._get_deployed_template( + stack_actions, is_stack_deployed + ) + + self._handle_special_parameter_situations( + stack_actions, generated_config, deployed_config + ) + + template_diff = self.compare_templates(deployed_template, generated_template) + config_diff = self.compare_stack_configurations( + deployed_config, generated_config + ) + + return StackDiff( + stack_actions.stack.external_name, + template_diff, + config_diff, + is_stack_deployed, + generated_config, + generated_template, + ) + + def _create_generated_config(self, stack: Stack) -> StackConfiguration: + parameters = self._extract_parameters_from_generated_stack(stack) + stack_configuration = StackConfiguration( + stack_name=stack.external_name, + parameters=parameters, + stack_tags=stack.tags, + notifications=stack.notifications, + cloudformation_service_role=stack.cloudformation_service_role, + ) + + return stack_configuration + + def _extract_parameters_from_generated_stack(self, stack: Stack) -> dict: + """Extracts a usable dict of parameters from the stack, performing some minor transformations + to match the what CloudFormation does on its end. + + :param stack: The stack to extract the parameters from + :return: A dictionary of stack parameters to be compared. + """ + formatted_parameters = {} + for key, value in stack.parameters.items(): + if isinstance(value, list): + value = ",".join(item.rstrip("\n") for item in value) + formatted_parameters[key] = value.rstrip("\n") + + return formatted_parameters + + def _create_deployed_stack_config( + self, stack_actions: StackActions + ) -> Optional[StackConfiguration]: + description = stack_actions.describe() + if description is None: + # This means the stack has not been deployed yet + return None + + stacks = description["Stacks"] + for stack in stacks: + if stack["StackStatus"] in self.STACK_STATUSES_INDICATING_NOT_DEPLOYED: + return None + return StackConfiguration( + parameters={ + param["ParameterKey"]: param["ParameterValue"] + for param in stack.get("Parameters", []) + }, + stack_tags={tag["Key"]: tag["Value"] for tag in stack["Tags"]}, + stack_name=stack["StackName"], + notifications=stack["NotificationARNs"], + cloudformation_service_role=stack.get("RoleARN"), + ) + + def _handle_special_parameter_situations( + self, + stack_actions: StackActions, + generated_config: StackConfiguration, + deployed_config: StackConfiguration, + ): + deployed_template_summary = stack_actions.fetch_remote_template_summary() + generated_template_summary = stack_actions.fetch_local_template_summary() + + if deployed_config is not None: + # Trailing linebreaks sometimes get removed by CloudFormation in certain circumstances + # and can sometimes come from using the !file_contents resolver, but ultimately they + # shouldn't affect the diff. We'll ignore all trailing linebreaks. + self._remove_terminating_linebreaks_from_deployed_parameters( + deployed_template_summary, deployed_config + ) + + # If the parameter is not passed by Sceptre and the value on the deployed parameter is + # the default value, we'll actually remove it from the deployed parameters list so it + # doesn't show up as a false positive. + self._remove_deployed_default_parameters_that_arent_passed( + deployed_template_summary, generated_config, deployed_config + ) + if not self.show_no_echo: + # We don't actually want to show parameters Sceptre is passing that the local template + # marks as NoEcho parameters (unless show_no_echo is set to true). Therefore those + # parameter values will be masked. + self._mask_no_echo_parameters(generated_template_summary, generated_config) + + def _remove_terminating_linebreaks_from_deployed_parameters( + self, template_summary: Optional[dict], deployed_config: StackConfiguration + ): + if template_summary is None: + return + + parameter_types = { + parameter["ParameterKey"]: parameter["ParameterType"] + for parameter in template_summary["Parameters"] + } + + for key, value in deployed_config.parameters.items(): + parameter_type = parameter_types[key] + if parameter_type == "CommaDelimitedList": + # If it's a list of strings, remove trailing linebreaks for each item + value = ",".join([item.rstrip("\n") for item in value.split(",")]) + + deployed_config.parameters[key] = value.rstrip("\n") + + def _remove_deployed_default_parameters_that_arent_passed( + self, + template_summary: dict, + generated_config: StackConfiguration, + deployed_config: StackConfiguration, + ): + deployed_config_default_map = self._get_parameter_default_map(template_summary) + for parameter_key, default_value in deployed_config_default_map.items(): + # If the generated config defines that parameter, leave that in the deployed config + # so we can see the diff. + if parameter_key in generated_config.parameters: + continue + + # But if the stack config is relying on the template's default value for this parameter, + # remove that parameter from the deployed config... but only if its value is set to the + # default value. If we don't do this, it will show up as a difference when it isn't. + if deployed_config.parameters[parameter_key] == default_value: + del deployed_config.parameters[parameter_key] + + def _get_parameter_default_map(self, template_summary: dict) -> Dict[str, str]: + if template_summary is None: + return {} + + parameters = template_summary["Parameters"] + default_map = {} + + for parameter in parameters: + key = parameter["ParameterKey"] + value = self._handle_default_value(parameter) + if value is not None: + default_map[key] = value + + return default_map + + def _handle_default_value(self, parameter): + default_value = parameter.get("DefaultValue") + param_type = parameter["ParameterType"] + + if default_value is None: + return None + + if parameter.get("NoEcho"): + default_value = "****" + elif "List" in param_type: + # Eliminate whitespace around commas + default_value = ",".join( + value.strip() for value in default_value.split(",") + ) + + return default_value + + def _mask_no_echo_parameters( + self, template_summary: dict, generated_config: StackConfiguration + ): + parameters = template_summary["Parameters"] + + for parameter in parameters: + key = parameter["ParameterKey"] + if parameter.get("NoEcho") and key in generated_config.parameters: + generated_config.parameters[key] = self.NO_ECHO_REPLACEMENT + + def _generate_template(self, stack_actions: StackActions) -> str: + return stack_actions.generate() + + def _get_deployed_template( + self, stack_actions: StackActions, is_deployed: bool + ) -> str: + if is_deployed: + return stack_actions.fetch_remote_template() or "{}" + else: + return "{}" + + @abstractmethod + def compare_templates(self, deployed: str, generated: str) -> DiffType: + """Implement this method to return the diff for the templates + + :param deployed: The stack template as it has been deployed + :param generated: The stack template as it exists locally within Sceptre + :return: The generated diff between the two + """ + + @abstractmethod + def compare_stack_configurations( + self, deployed: Optional[StackConfiguration], generated: StackConfiguration + ) -> DiffType: + """Implement this method to return the diff for the stack configurations. + + :param deployed: The StackConfiguration as it has been deployed. This MIGHT be None, if the + stack has not been deployed. + :param generated: The StackConfiguration as it exists locally within Sceptre + :return: The generated diff between the two + """ + + +class DeepDiffStackDiffer(StackDiffer[deepdiff.DeepDiff]): + """A StackDiffer that relies upon the DeepDiff library to produce the difference between the + stack as it has been deployed onto CloudFormation and as it exists locally within Sceptre. + + This differ relies upon a recursive key/value comparison of Python data structures, indicating + specific keys or values that have been added, removed, or altered between the two. Templates + are read in as dictionaries and compared this way, so json or yaml formatting changes will not + be reflected, only changes in value. + """ + + VERBOSITY_LEVEL_TO_INDICATE_CHANGED_VALUES = 2 + + def __init__( + self, + show_no_echo=False, + *, + universal_template_loader: Callable[[str], Tuple[dict, str]] = cfn_flip.load + ): + """Initializes a DeepDiffStackDiffer. + + :param show_no_echo: If True, local parameters passed that the template says are NoEcho + parameters will be displayed; Otherwise, they will be masked in the diff. + :param universal_template_loader: This should be a callable that can load either a json or + yaml string and return a tuple where the first element is the loaded template and the + second element is the template format (either "json" or "yaml") + """ + super().__init__(show_no_echo) + self.load_template = universal_template_loader + + def compare_stack_configurations( + self, + deployed: Optional[StackConfiguration], + generated: StackConfiguration, + ) -> deepdiff.DeepDiff: + return deepdiff.DeepDiff( + deployed, + generated, + verbose_level=self.VERBOSITY_LEVEL_TO_INDICATE_CHANGED_VALUES, + ) + + def compare_templates(self, deployed: str, generated: str) -> deepdiff.DeepDiff: + # We don't actually care about the original formats here, since we only care about the + # template VALUES. + deployed_dict, _ = self.load_template(deployed) + generated_dict, _ = self.load_template(generated) + + return deepdiff.DeepDiff( + deployed_dict, + generated_dict, + verbose_level=self.VERBOSITY_LEVEL_TO_INDICATE_CHANGED_VALUES, + ) + + +class DifflibStackDiffer(StackDiffer[List[str]]): + """A StackDiffer that uses difflib to produce a diff between the stack as it exists on AWS and + the stack as it exists locally within Sceptre. + + Because difflib generates diffs off of lists of strings, both StackConfigurations and + """ + + def __init__( + self, + show_no_echo=False, + *, + universal_template_loader: Callable[[str], Tuple[dict, str]] = cfn_flip.load + ): + """Initializes a DifflibStackDiffer. + + :param show_no_echo: If True, local parameters passed that the template says are NoEcho + parameters will be displayed; Otherwise, they will be masked in the diff. + :param universal_template_loader: This should be a callable that can load either a json or + yaml string and return a tuple where the first element is the loaded template and the + second element is the template format (either "json" or "yaml") + """ + super().__init__(show_no_echo) + self.load_template = universal_template_loader + + def compare_stack_configurations( + self, + deployed: Optional[StackConfiguration], + generated: StackConfiguration, + ) -> List[str]: + if deployed is None: + comparable_deployed = None + else: + comparable_deployed = self._make_stack_configuration_comparable(deployed) + + comparable_generated = self._make_stack_configuration_comparable(generated) + deployed_string = cfn_flip.dump_yaml(comparable_deployed) + generated_string = cfn_flip.dump_yaml(comparable_generated) + return self._make_string_diff(deployed_string, generated_string) + + def _make_stack_configuration_comparable( + self, config: Optional[StackConfiguration] + ): + as_dict = dict(config._asdict()) + return { + key: value + for key, value in as_dict.items() + # stack_name isn't always going to be the same, otherwise we wouldn't be comparing them. + # It's more confusing to have it in the diff output than to just remove it. + if value not in (None, [], {}) and key != "stack_name" + } + + def compare_templates( + self, + deployed: str, + generated: str, + ) -> List[str]: + # Sometimes there might only be simple whitespace differences... which difflib will show but + # are actually insignificant and "false positives". Also, it's POSSIBLE that the template + # format might have changed, even if all the VALUES have stayed the same, so we'll read both + # templates into dicts (regardless of their format) and we'll output both to the format of + # the local template using identical serialization settings. This will truly enable comparison + # of the actual values rather than other things that don't actually make a difference to + # CloudFormation. If only the format/meaningless whitespace has changed, this will result in + # there being no diff. + deployed_dict, _ = self.load_template(deployed) + generated_dict, generated_format = self.load_template(generated) + dumpers = {"json": cfn_flip.dump_json, "yaml": cfn_flip.dump_yaml} + deployed_reformatted = dumpers[generated_format](deployed_dict) + generated_reformatted = dumpers[generated_format](generated_dict) + + return self._make_string_diff(deployed_reformatted, generated_reformatted) + + def _make_string_diff(self, deployed: str, generated: str) -> List[str]: + diff_lines = difflib.unified_diff( + deployed.splitlines(), + generated.splitlines(), + fromfile="deployed", + tofile="generated", + lineterm="", + ) + return list(diff_lines) diff --git a/sceptre/exceptions.py b/sceptre/exceptions.py index 75438d9e0..a0ebce583 100644 --- a/sceptre/exceptions.py +++ b/sceptre/exceptions.py @@ -5,6 +5,7 @@ class SceptreException(Exception): """ Base class for all Sceptre errors """ + pass @@ -12,6 +13,7 @@ class ProjectAlreadyExistsError(SceptreException): """ Error raised when Sceptre project already exists. """ + pass @@ -19,6 +21,7 @@ class InvalidSceptreDirectoryError(SceptreException): """ Error raised if a sceptre directory is invalid. """ + pass @@ -26,6 +29,7 @@ class UnsupportedTemplateFileTypeError(SceptreException): """ Error raised if an unsupported template file type is used. """ + pass @@ -33,6 +37,7 @@ class TemplateSceptreHandlerError(SceptreException): """ Error raised if sceptre_handler() is not defined correctly in the template. """ + pass @@ -40,6 +45,7 @@ class DependencyDoesNotExistError(SceptreException): """ Error raised when a dependency cannot be found """ + pass @@ -47,6 +53,7 @@ class DependencyStackNotLaunchedError(SceptreException): """ Error raised when a dependency stack has not been launched """ + pass @@ -54,6 +61,7 @@ class DependencyStackMissingOutputError(SceptreException): """ Error raised if a dependency stack does not have the correct outputs. """ + pass @@ -61,6 +69,7 @@ class CircularDependenciesError(SceptreException): """ Error raised if there are circular dependencies """ + pass @@ -68,6 +77,7 @@ class UnknownStackStatusError(SceptreException): """ Error raised if an unknown stack status is received. """ + pass @@ -75,6 +85,7 @@ class RetryLimitExceededError(SceptreException): """ Error raised if the request limit is exceeded. """ + pass @@ -88,6 +99,7 @@ class VersionIncompatibleError(SceptreException): """ Error raised if configuration incompatible with running version. """ + pass @@ -95,6 +107,7 @@ class ProtectedStackError(SceptreException): """ Error raised upon execution of an action under active protection """ + pass @@ -102,6 +115,7 @@ class UnknownStackChangeSetStatusError(SceptreException): """ Error raised if an unknown stack change set status is received. """ + pass @@ -109,6 +123,7 @@ class InvalidHookArgumentTypeError(SceptreException): """ Error raised if a hook's argument type is invalid. """ + pass @@ -116,6 +131,7 @@ class InvalidHookArgumentSyntaxError(SceptreException): """ Error raised if a hook's argument syntax is invalid. """ + pass @@ -123,6 +139,7 @@ class InvalidHookArgumentValueError(SceptreException): """ Error raised if a hook's argument value is invalid. """ + pass @@ -130,6 +147,7 @@ class CannotUpdateFailedStackError(SceptreException): """ Error raised when a failed stack is updated. """ + pass @@ -137,6 +155,7 @@ class StackDoesNotExistError(SceptreException): """ Error raised when a stack does not exist. """ + pass @@ -144,6 +163,7 @@ class ConfigFileNotFoundError(SceptreException): """ Error raised when a config file does not exist. """ + pass @@ -151,6 +171,7 @@ class InvalidConfigFileError(SceptreException): """ Error raised when a config file lacks mandatory keys. """ + pass @@ -158,6 +179,7 @@ class PathConversionError(SceptreException): """ Error raised when a path is unable to be converted. """ + pass @@ -165,4 +187,41 @@ class InvalidAWSCredentialsError(SceptreException): """ Error raised when AWS credentials are invalid. """ + pass + + +class TemplateHandlerNotFoundError(SceptreException): + """ + Error raised when a Template Handler of a certain type is not found + """ + + pass + + +class TemplateHandlerArgumentsInvalidError(SceptreException): + """ + Error raised when the arguments passed to a Template Handler do not + adhere to the specified JSON schema. + """ + + +class TemplateNotFoundError(SceptreException): + """ + Error raised when a Template file is not found + """ + + pass + + +class CannotPruneStackError(SceptreException): + """ + Error raised when an obsolete stack cannot be pruned because another stack depends on it that is + not itself obsolete. + """ + + +class InvalidResolverArgumentError(SceptreException): + """ + Indicates a resolver argument is invalid in some way. + """ diff --git a/sceptre/helpers.py b/sceptre/helpers.py index 22eefce6e..66b8c6a0a 100644 --- a/sceptre/helpers.py +++ b/sceptre/helpers.py @@ -1,7 +1,14 @@ # -*- coding: utf-8 -*- +from contextlib import contextmanager +from datetime import datetime from os import sep +from typing import Optional, Any, List, Tuple, Union + +import dateutil.parser +import deprecation from sceptre.exceptions import PathConversionError +from sceptre import __version__ def get_external_stack_name(project_code, stack_name): @@ -14,10 +21,7 @@ def get_external_stack_name(project_code, stack_name): :returns: The name given to the stack in CloudFormation. :rtype: str """ - return "-".join([ - project_code, - stack_name.replace("/", "-") - ]) + return "-".join([project_code, stack_name.replace("/", "-")]) def mask_key(key): @@ -34,10 +38,7 @@ def mask_key(key): """ num_mask_chars = len(key) - 4 - return "".join([ - "*" if i < num_mask_chars else c - for i, c in enumerate(key) - ]) + return "".join(["*" if i < num_mask_chars else c for i, c in enumerate(key)]) def _call_func_on_values(func, attr, cls): @@ -67,6 +68,28 @@ def func_on_instance(key): return attr +Container = Union[list, dict] +Key = Union[str, int] + + +def delete_keys_from_containers(keys_to_delete: List[Tuple[Container, Key]]): + """Removes the indicated keys/indexes from their paired containers.""" + list_items_to_delete = [] + for container, key in keys_to_delete: + if isinstance(container, list): + # If it's a list, we want to gather up the items to remove from the list. + # We don't want to modify the list length yet, since removals will change all the other + # list indexes. Instead, we'll get the actual items at those indexes to remove later. + list_items_to_delete.append((container, container[key])) + else: + del container[key] + + # Finally, now that we have all the items we want to remove the lists, we'll remove those + # items specifically from the lists. + for containing_list, item in list_items_to_delete: + containing_list.remove(item) + + def normalise_path(path): """ Converts a path to use correct path separator. @@ -78,10 +101,10 @@ def normalise_path(path): :returns: A normalised path with forward slashes. :returns: string """ - if sep is '/': - path = path.replace('\\', '/') - elif sep is '\\': - path = path.replace('/', '\\') + if sep == "/": + path = path.replace("\\", "/") + elif sep == "\\": + path = path.replace("/", "\\") if path.endswith("/") or path.endswith("\\"): raise PathConversionError( "'{0}' is an invalid path string. Paths should " @@ -101,10 +124,96 @@ def sceptreise_path(path): :returns: A normalised path with forward slashes. :returns: string """ - path = path.replace('\\', '/') + path = path.replace("\\", "/") if path.endswith("/") or path.endswith("\\"): raise PathConversionError( "'{0}' is an invalid path string. Paths should " "not have trailing slashes.".format(path) ) return path + + +@contextmanager +def null_context(): + """A context manager that does nothing. This is identical to the nullcontext in py3.7+, but isn't + available in py3.6, so providing it here instead. + """ + yield + + +def extract_datetime_from_aws_response_headers( + boto_response: dict, +) -> Optional[datetime]: + """Returns a datetime.datetime extracted from the response metadata in a + boto response or None if it's unable to find or parse one. + :param boto_response: A dictionary returned from a boto client call + :returns a datetime.datetime or None + """ + if boto_response is None: + return None + try: + return dateutil.parser.parse( + boto_response["ResponseMetadata"]["HTTPHeaders"]["date"] + ) + except (KeyError, dateutil.parser.ParserError): + # We expect a KeyError if the date isn't present in the response. We + # expect a ParserError if it's not well-formed. Any other error we want + # to pass along. + return None + + +def gen_repr(instance: Any, class_label: str = None, attributes: List[str] = []) -> str: + """ + Returns a standard __repr__ based on instance attributes. + :param instance: The instance to represent (`self`). + :param class_label: Override the name of the class found through introspection. + :param attributes: List the attributes to include the in representation. + :returns: A string representation of `instance` + """ + if not class_label: + class_label = instance.__class__.__name__ + attr_str = ", ".join( + [f"{a}={repr(instance.__getattribute__(a))}" for a in attributes] + ) + return f"{class_label}({attr_str})" + + +def create_deprecated_alias_property( + alias_from: str, alias_to: str, deprecated_in: str, removed_in: Optional[str] +) -> property: + """Creates a property object with a deprecated getter and a deprecated setter that emit warnings + when used, aliasing to their renamed property names. + + :param alias_from: The name of the attribute that is deprecated and that needs to be aliased + :param alias_to: The name of the attribute to alias the deprecated field to. + :param deprecated_in: The version in which the property is deprecated. + :param removed_in: The version when it will be removed, after which the alias will no longer work. + This value can be None, indicating that removal is not yet planned. + :return: A property object to be assigned directly onto a class. + """ + + def getter(self): + return getattr(self, alias_to) + + getter.__name__ = alias_from + + def setter(self, value): + setattr(self, alias_to, value) + + setter.__name__ = alias_from + + deprecation_kwargs = dict( + deprecated_in=deprecated_in, + removed_in=removed_in, + current_version=__version__, + details=( + f'It is being renamed to "{alias_to}". You should migrate all uses of "{alias_from}" to ' + f"that in order to avoid future breakage." + ), + ) + + deprecated_getter = deprecation.deprecated(**deprecation_kwargs)(getter) + deprecated_setter = deprecation.deprecated(**deprecation_kwargs)(setter) + + deprecated_property = property(deprecated_getter, deprecated_setter) + return deprecated_property diff --git a/sceptre/hooks/__init__.py b/sceptre/hooks/__init__.py index e987d336c..56b4df68a 100644 --- a/sceptre/hooks/__init__.py +++ b/sceptre/hooks/__init__.py @@ -1,32 +1,21 @@ import abc import logging from functools import wraps +from typing import TYPE_CHECKING from sceptre.helpers import _call_func_on_values +from sceptre.resolvers import CustomYamlTagBase +if TYPE_CHECKING: + from sceptre.stack import Stack -class Hook(object): - """ - Hook is an abstract base class that should be inherited by all hooks. - :param argument: The argument of the hook. - :type argument: str - :param stack: The associated stack of the hook. - :type stack: sceptre.stack.Stack +class Hook(CustomYamlTagBase, metaclass=abc.ABCMeta): + """ + Hook is an abstract base class that should be subclassed by all hooks. """ - __metaclass__ = abc.ABCMeta - def __init__(self, argument=None, stack=None): - self.logger = logging.getLogger(__name__) - self.argument = argument - self.stack = stack - - def setup(self): - """ - setup is a method that may be overwritten by inheriting classes. Allows - hooks to run so initalisation steps when config is first read. - """ - pass # pragma: no cover + logger = logging.getLogger(__name__) @abc.abstractmethod def run(self): @@ -60,15 +49,16 @@ def __get__(self, instance, type): """ return getattr(instance, self.name) - def __set__(self, instance, value): + def __set__(self, instance: "Stack", value): """ Attribute setter which adds a stack reference to any hooks in the data structure `value` and calls the setup method. """ - def setup(attr, key, value): - value.stack = instance - value.setup() + + def setup(attr, key, value: Hook): + attr[key] = clone = value.clone_for_stack(instance) + clone.setup() _call_func_on_values(setup, value, Hook) setattr(instance, self.name, value) @@ -98,6 +88,7 @@ def add_stack_hooks(func): :param func: a function that operates on a stack :type func: function """ + @wraps(func) def decorated(self, *args, **kwargs): execute_hooks(self.stack.hooks.get("before_" + func.__name__)) diff --git a/sceptre/hooks/asg_scaling_processes.py b/sceptre/hooks/asg_scaling_processes.py index f377265c5..0ea3c263a 100644 --- a/sceptre/hooks/asg_scaling_processes.py +++ b/sceptre/hooks/asg_scaling_processes.py @@ -30,14 +30,15 @@ def run(self): if not isinstance(self.argument, string_types): raise InvalidHookArgumentTypeError( 'The argument "{0}" is the wrong type - asg_scaling_processes ' - 'hooks require arguments of type string.'.format(self.argument) + "hooks require arguments of type string.".format(self.argument) ) if "::" not in str(self.argument): raise InvalidHookArgumentSyntaxError( 'Wrong syntax for the argument "{0}" - asg_scaling_processes ' - 'hooks use:' - '- !asg_scaling_processes ::' - .format(self.argument) + "hooks use:" + "- !asg_scaling_processes ::".format( + self.argument + ) ) action, scaling_processes = self.argument.split("::") @@ -45,8 +46,7 @@ def run(self): if action not in ["resume", "suspend"]: raise InvalidHookArgumentValueError( 'The argument "{0}" is invalid - valid arguments for ' - 'asg_scaling_processes hooks are "resume" or "suspend".' - .format(action) + 'asg_scaling_processes hooks are "resume" or "suspend".'.format(action) ) action += "_processes" @@ -58,8 +58,8 @@ def run(self): command=action, kwargs={ "AutoScalingGroupName": autoscaling_group, - "ScalingProcesses": [scaling_processes] - } + "ScalingProcesses": [scaling_processes], + }, ) def _get_stack_resources(self): @@ -70,7 +70,7 @@ def _get_stack_resources(self): response = self.stack.connection_manager.call( service="cloudformation", command="describe_stack_resources", - kwargs={"StackName": self.stack.external_name} + kwargs={"StackName": self.stack.external_name}, ) return response.get("StackResources", []) diff --git a/sceptre/hooks/cmd.py b/sceptre/hooks/cmd.py index 4cf6a1f8b..5ec32165e 100644 --- a/sceptre/hooks/cmd.py +++ b/sceptre/hooks/cmd.py @@ -18,10 +18,11 @@ def run(self): :raises: sceptre.exceptions.InvalidTaskArgumentTypeException :raises: subprocess.CalledProcessError """ + envs = self.stack.connection_manager.create_session_environment_variables() try: - subprocess.check_call(self.argument, shell=True) + subprocess.check_call(self.argument, shell=True, env=envs) except TypeError: raise InvalidHookArgumentTypeError( 'The argument "{0}" is the wrong type - cmd hooks require ' - 'arguments of type string.'.format(self.argument) + "arguments of type string.".format(self.argument) ) diff --git a/sceptre/logging.py b/sceptre/logging.py new file mode 100644 index 000000000..56e9643f5 --- /dev/null +++ b/sceptre/logging.py @@ -0,0 +1,20 @@ +from logging import LoggerAdapter, Logger +from typing import MutableMapping, Any, Tuple + + +class StackLoggerAdapter(LoggerAdapter): + def __init__(self, logger: Logger, stack_name: str, extra: dict = None): + """A small wrapper around a Logger that prefixes log messages with the stack name. + + :param logger: The logger to wrap + :param stack_name: The name of the stack to every log message + :param extra: Extra kwargs to add to the log context (if any) + """ + super().__init__(logger, extra or {}) + self.stack_name = stack_name + + def process( + self, msg: str, kwargs: MutableMapping[str, Any] + ) -> Tuple[Any, MutableMapping[str, Any]]: + msg = f"{self.stack_name} - {msg}" + return super().process(msg, kwargs) diff --git a/sceptre/plan/__init__.py b/sceptre/plan/__init__.py index 1656562d8..4f8bf5472 100644 --- a/sceptre/plan/__init__.py +++ b/sceptre/plan/__init__.py @@ -3,8 +3,8 @@ import logging -__author__ = 'Cloudreach' -__email__ = 'sceptre@cloudreach.com' +__author__ = "Cloudreach" +__email__ = "sceptre@cloudreach.com" # Set up logging to ``/dev/null`` like a library is supposed to. @@ -14,4 +14,4 @@ def emit(self, record): pass -logging.getLogger('sceptre').addHandler(NullHandler()) +logging.getLogger("sceptre").addHandler(NullHandler()) diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 575126ae6..d8bb02cee 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -7,29 +7,37 @@ available to a Stack. """ +import json import logging import time - -from os import path +import typing +import urllib from datetime import datetime, timedelta +from os import path +from typing import Dict, Optional, Tuple, Union import botocore -import json from dateutil.tz import tzutc +from sceptre.config.reader import ConfigReader from sceptre.connection_manager import ConnectionManager +from sceptre.exceptions import ( + CannotUpdateFailedStackError, + ProtectedStackError, + StackDoesNotExistError, + UnknownStackChangeSetStatusError, + UnknownStackStatusError, +) +from sceptre.helpers import extract_datetime_from_aws_response_headers, normalise_path from sceptre.hooks import add_stack_hooks -from sceptre.stack_status import StackStatus -from sceptre.stack_status import StackChangeSetStatus +from sceptre.stack import Stack +from sceptre.stack_status import StackChangeSetStatus, StackStatus -from sceptre.exceptions import CannotUpdateFailedStackError -from sceptre.exceptions import UnknownStackStatusError -from sceptre.exceptions import UnknownStackChangeSetStatusError -from sceptre.exceptions import StackDoesNotExistError -from sceptre.exceptions import ProtectedStackError +if typing.TYPE_CHECKING: + from sceptre.diffing.stack_differ import StackDiff, StackDiffer -class StackActions(object): +class StackActions: """ StackActions stores the operations a Stack can take, such as creating or deleting the Stack. @@ -38,13 +46,16 @@ class StackActions(object): :type stack: sceptre.stack.Stack """ - def __init__(self, stack): + def __init__(self, stack: Stack): self.stack = stack self.name = self.stack.name self.logger = logging.getLogger(__name__) self.connection_manager = ConnectionManager( - self.stack.region, self.stack.profile, - self.stack.external_name, self.stack.iam_role + self.stack.region, + self.stack.profile, + self.stack.external_name, + self.stack.sceptre_role, + self.stack.sceptre_role_session_duration, ) @add_stack_hooks @@ -60,18 +71,24 @@ def create(self): create_stack_kwargs = { "StackName": self.stack.external_name, "Parameters": self._format_parameters(self.stack.parameters), - "Capabilities": ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND'], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], "NotificationARNs": self.stack.notifications, "Tags": [ - {"Key": str(k), "Value": str(v)} - for k, v in self.stack.tags.items() - ] + {"Key": str(k), "Value": str(v)} for k, v in self.stack.tags.items() + ], } - if self.stack.on_failure: + # can specify either DisableRollback or OnFailure , but not both + if self.stack.disable_rollback: + create_stack_kwargs.update({"DisableRollback": self.stack.disable_rollback}) + elif self.stack.on_failure: create_stack_kwargs.update({"OnFailure": self.stack.on_failure}) - create_stack_kwargs.update( - self.stack.template.get_boto_call_parameter()) + + create_stack_kwargs.update(self.stack.template.get_boto_call_parameter()) create_stack_kwargs.update(self._get_role_arn()) create_stack_kwargs.update(self._get_stack_timeout()) @@ -79,21 +96,19 @@ def create(self): response = self.connection_manager.call( service="cloudformation", command="create_stack", - kwargs=create_stack_kwargs + kwargs=create_stack_kwargs, ) self.logger.debug( "%s - Create stack response: %s", self.stack.name, response ) - status = self._wait_for_completion() + status = self._wait_for_completion(boto_response=response) except botocore.exceptions.ClientError as exp: if exp.response["Error"]["Code"] == "AlreadyExistsException": - self.logger.info( - "%s - Stack already exists", self.stack.name - ) + self.logger.info("%s - Stack already exists", self.stack.name) - status = "COMPLETE" + status = StackStatus.COMPLETE else: raise @@ -114,25 +129,25 @@ def update(self): "StackName": self.stack.external_name, "Parameters": self._format_parameters(self.stack.parameters), "Capabilities": [ - 'CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND' + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", ], "NotificationARNs": self.stack.notifications, "Tags": [ - {"Key": str(k), "Value": str(v)} - for k, v in self.stack.tags.items() - ] + {"Key": str(k), "Value": str(v)} for k, v in self.stack.tags.items() + ], } - update_stack_kwargs.update( - self.stack.template.get_boto_call_parameter()) + update_stack_kwargs.update(self.stack.template.get_boto_call_parameter()) update_stack_kwargs.update(self._get_role_arn()) response = self.connection_manager.call( service="cloudformation", command="update_stack", - kwargs=update_stack_kwargs + kwargs=update_stack_kwargs, + ) + status = self._wait_for_completion( + self.stack.stack_timeout, boto_response=response ) - status = self._wait_for_completion(self.stack.stack_timeout) self.logger.debug( "%s - Update Stack response: %s", self.stack.name, response ) @@ -145,9 +160,7 @@ def update(self): except botocore.exceptions.ClientError as exp: error_message = exp.response["Error"]["Message"] if error_message == "No updates are to be performed.": - self.logger.info( - "%s - No updates to perform.", self.stack.name - ) + self.logger.info("%s - No updates to perform.", self.stack.name) return StackStatus.COMPLETE else: raise @@ -160,20 +173,20 @@ def cancel_stack_update(self): :rtype: sceptre.stack_status.StackStatus """ self.logger.warning( - "%s - Update Stack time exceeded the specified timeout", - self.stack.name + "%s - Update Stack time exceeded the specified timeout", self.stack.name ) response = self.connection_manager.call( service="cloudformation", command="cancel_update_stack", - kwargs={"StackName": self.stack.external_name} + kwargs={"StackName": self.stack.external_name}, ) self.logger.debug( "%s - Cancel update Stack response: %s", self.stack.name, response ) - return self._wait_for_completion() + return self._wait_for_completion(boto_response=response) - def launch(self): + @add_stack_hooks + def launch(self) -> StackStatus: """ Launches the Stack. @@ -183,10 +196,10 @@ def launch(self): performed, launch exits gracefully. :returns: The Stack's status. - :rtype: sceptre.stack_status.StackStatus """ self._protect_execution() - self.logger.info("%s - Launching Stack", self.stack.name) + self.logger.info(f"{self.stack.name} - Launching Stack") + try: existing_status = self._get_status() except StackDoesNotExistError: @@ -206,20 +219,18 @@ def launch(self): elif existing_status.endswith("IN_PROGRESS"): self.logger.info( "%s - Stack action is already in progress state and cannot " - "be updated", self.stack.name + "be updated", + self.stack.name, ) status = StackStatus.IN_PROGRESS elif existing_status.endswith("FAILED"): - status = StackStatus.FAILED raise CannotUpdateFailedStackError( "'{0}' is in a the state '{1}' and cannot be updated".format( self.stack.name, existing_status ) ) else: - raise UnknownStackStatusError( - "{0} is unknown".format(existing_status) - ) + raise UnknownStackStatusError("{0} is unknown".format(existing_status)) return status @add_stack_hooks @@ -242,14 +253,12 @@ def delete(self): delete_stack_kwargs = {"StackName": self.stack.external_name} delete_stack_kwargs.update(self._get_role_arn()) - self.connection_manager.call( - service="cloudformation", - command="delete_stack", - kwargs=delete_stack_kwargs + response = self.connection_manager.call( + service="cloudformation", command="delete_stack", kwargs=delete_stack_kwargs ) try: - status = self._wait_for_completion() + status = self._wait_for_completion(boto_response=response) except StackDoesNotExistError: status = StackStatus.COMPLETE except botocore.exceptions.ClientError as error: @@ -268,7 +277,7 @@ def lock(self): # need to get to the base install path. __file__ will take us into # sceptre/actions so need to walk up the path. path.abspath(path.join(__file__, "..", "..")), - "stack_policies/lock.json" + "stack_policies/lock.json", ) self.set_policy(policy_path) self.logger.info("%s - Successfully locked Stack", self.stack.name) @@ -281,7 +290,7 @@ def unlock(self): # need to get to the base install path. __file__ will take us into # sceptre/actions so need to walk up the path. path.abspath(path.join(__file__, "..", "..")), - "stack_policies/unlock.json" + "stack_policies/unlock.json", ) self.set_policy(policy_path) self.logger.info("%s - Successfully unlocked Stack", self.stack.name) @@ -297,7 +306,7 @@ def describe(self): return self.connection_manager.call( service="cloudformation", command="describe_stacks", - kwargs={"StackName": self.stack.external_name} + kwargs={"StackName": self.stack.external_name}, ) except botocore.exceptions.ClientError as e: if e.response["Error"]["Message"].endswith("does not exist"): @@ -314,7 +323,7 @@ def describe_events(self): return self.connection_manager.call( service="cloudformation", command="describe_stack_events", - kwargs={"StackName": self.stack.external_name} + kwargs={"StackName": self.stack.external_name}, ) def describe_resources(self): @@ -329,7 +338,7 @@ def describe_resources(self): response = self.connection_manager.call( service="cloudformation", command="describe_stack_resources", - kwargs={"StackName": self.stack.external_name} + kwargs={"StackName": self.stack.external_name}, ) except botocore.exceptions.ClientError as e: if e.response["Error"]["Message"].endswith("does not exist"): @@ -337,17 +346,17 @@ def describe_resources(self): raise self.logger.debug( - "%s - Describe Stack resource response: %s", - self.stack.name, - response + "%s - Describe Stack resource response: %s", self.stack.name, response ) desired_properties = ["LogicalResourceId", "PhysicalResourceId"] - formatted_response = {self.stack.name: [ - {k: v for k, v in item.items() if k in desired_properties} - for item in response["StackResources"] - ]} + formatted_response = { + self.stack.name: [ + {k: v for k, v in item.items() if k in desired_properties} + for item in response["StackResources"] + ] + } return formatted_response def describe_outputs(self): @@ -372,18 +381,16 @@ def continue_update_rollback(self): UPDATE_ROLLBACK_COMPLETE. """ self.logger.debug("%s - Continuing update rollback", self.stack.name) - continue_update_rollback_kwargs = { - "StackName": self.stack.external_name - } + continue_update_rollback_kwargs = {"StackName": self.stack.external_name} continue_update_rollback_kwargs.update(self._get_role_arn()) self.connection_manager.call( service="cloudformation", command="continue_update_rollback", - kwargs=continue_update_rollback_kwargs + kwargs=continue_update_rollback_kwargs, ) self.logger.info( "%s - Successfully initiated continuation of update rollback", - self.stack.name + self.stack.name, ) def set_policy(self, policy_path): @@ -397,19 +404,12 @@ def set_policy(self, policy_path): with open(policy_path) as f: policy = f.read() - self.logger.debug( - "%s - Setting Stack policy: \n%s", - self.stack.name, - policy - ) + self.logger.debug("%s - Setting Stack policy: \n%s", self.stack.name, policy) self.connection_manager.call( service="cloudformation", command="set_stack_policy", - kwargs={ - "StackName": self.stack.external_name, - "StackPolicyBody": policy - } + kwargs={"StackName": self.stack.external_name, "StackPolicyBody": policy}, ) self.logger.info("%s - Successfully set Stack Policy", self.stack.name) @@ -424,14 +424,14 @@ def get_policy(self): response = self.connection_manager.call( service="cloudformation", command="get_stack_policy", - kwargs={ - "StackName": self.stack.external_name - } + kwargs={"StackName": self.stack.external_name}, + ) + json_formatting = json.loads( + response.get("StackPolicyBody", json.dumps("No Policy Information")) ) - json_formatting = json.loads(response.get( - "StackPolicyBody", json.dumps("No Policy Information"))) return {self.stack.name: json_formatting} + @add_stack_hooks def create_change_set(self, change_set_name): """ Creates a Change Set with the name ``change_set_name``. @@ -442,17 +442,18 @@ def create_change_set(self, change_set_name): create_change_set_kwargs = { "StackName": self.stack.external_name, "Parameters": self._format_parameters(self.stack.parameters), - "Capabilities": ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND'], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], "ChangeSetName": change_set_name, "NotificationARNs": self.stack.notifications, "Tags": [ - {"Key": str(k), "Value": str(v)} - for k, v in self.stack.tags.items() - ] + {"Key": str(k), "Value": str(v)} for k, v in self.stack.tags.items() + ], } - create_change_set_kwargs.update( - self.stack.template.get_boto_call_parameter() - ) + create_change_set_kwargs.update(self.stack.template.get_boto_call_parameter()) create_change_set_kwargs.update(self._get_role_arn()) self.logger.debug( "%s - Creating Change Set '%s'", self.stack.name, change_set_name @@ -460,13 +461,14 @@ def create_change_set(self, change_set_name): self.connection_manager.call( service="cloudformation", command="create_change_set", - kwargs=create_change_set_kwargs + kwargs=create_change_set_kwargs, ) # After the call successfully completes, AWS CloudFormation # starts creating the Change Set. self.logger.info( "%s - Successfully initiated creation of Change Set '%s'", - self.stack.name, change_set_name + self.stack.name, + change_set_name, ) def delete_change_set(self, change_set_name): @@ -484,14 +486,15 @@ def delete_change_set(self, change_set_name): command="delete_change_set", kwargs={ "ChangeSetName": change_set_name, - "StackName": self.stack.external_name - } + "StackName": self.stack.external_name, + }, ) # If the call successfully completes, AWS CloudFormation # successfully deleted the Change Set. self.logger.info( "%s - Successfully deleted Change Set '%s'", - self.stack.name, change_set_name + self.stack.name, + change_set_name, ) def describe_change_set(self, change_set_name): @@ -511,8 +514,8 @@ def describe_change_set(self, change_set_name): command="describe_change_set", kwargs={ "ChangeSetName": change_set_name, - "StackName": self.stack.external_name - } + "StackName": self.stack.external_name, + }, ) def execute_change_set(self, change_set_name): @@ -525,47 +528,110 @@ def execute_change_set(self, change_set_name): :rtype: str """ self._protect_execution() + change_set = self.describe_change_set(change_set_name) + status = change_set.get("Status") + reason = change_set.get("StatusReason") + if status == "FAILED" and self.change_set_creation_failed_due_to_no_changes( + reason + ): + self.logger.info( + "Skipping ChangeSet on Stack: {} - there are no changes".format( + change_set.get("StackName") + ) + ) + return 0 + self.logger.debug( "%s - Executing Change Set '%s'", self.stack.name, change_set_name ) - self.connection_manager.call( + response = self.connection_manager.call( service="cloudformation", command="execute_change_set", kwargs={ "ChangeSetName": change_set_name, - "StackName": self.stack.external_name - } + "StackName": self.stack.external_name, + }, ) - - status = self._wait_for_completion() + status = self._wait_for_completion(boto_response=response) return status - def list_change_sets(self): + def change_set_creation_failed_due_to_no_changes(self, reason: str) -> bool: + """Indicates the change set failed when it was created because there were actually + no changes introduced from the change set. + + :param reason: The reason reported by CloudFormation for the Change Set failure + """ + reason = reason.lower() + no_change_substrings = ( + "submitted information didn't contain changes", + "no updates are to be performed", # The reason returned for SAM templates + ) + + for substring in no_change_substrings: + if substring in reason: + return True + return False + + def list_change_sets(self, url=False): """ Lists the Stack's Change Sets. + :param url: Write out a console URL instead. + :type url: bool + :returns: The Stack's Change Sets. :rtype: dict or list """ + response = self._list_change_sets() + summaries = response.get("Summaries", []) + + if url: + summaries = self._convert_to_url(summaries) + + return {self.stack.name: summaries} + + def _list_change_sets(self): self.logger.debug("%s - Listing change sets", self.stack.name) try: - response = self.connection_manager.call( + return self.connection_manager.call( service="cloudformation", command="list_change_sets", - kwargs={ - "StackName": self.stack.external_name - } + kwargs={"StackName": self.stack.external_name}, ) - return {self.stack.name: response.get("Summaries", [])} except botocore.exceptions.ClientError: return [] + def _convert_to_url(self, summaries): + """ + Convert the list_change_sets response from + CloudFormation to a URL in the AWS Console. + """ + new_summaries = [] + + for summary in summaries: + stack_id = summary["StackId"] + change_set_id = summary["ChangeSetId"] + + region = self.stack.region + encoded = urllib.parse.urlencode( + {"stackId": stack_id, "changeSetId": change_set_id} + ) + + new_summaries.append( + f"https://{region}.console.aws.amazon.com/cloudformation/home?" + f"region={region}#/stacks/changesets/changes?{encoded}" + ) + + return new_summaries + + @add_stack_hooks def generate(self): """ Returns the Template for the Stack """ return self.stack.template.body + @add_stack_hooks def validate(self): """ Validates the Stack's CloudFormation Template. @@ -580,7 +646,7 @@ def validate(self): response = self.connection_manager.call( service="cloudformation", command="validate_template", - kwargs=self.stack.template.get_boto_call_parameter() + kwargs=self.stack.template.get_boto_call_parameter(), ) self.logger.debug( "%s - Validate Template response: %s", self.stack.name, response @@ -598,17 +664,15 @@ def estimate_cost(self): self.logger.debug("%s - Estimating template cost", self.stack.name) parameters = [ - {'ParameterKey': key, 'ParameterValue': value} + {"ParameterKey": key, "ParameterValue": value} for key, value in self.stack.parameters.items() ] kwargs = self.stack.template.get_boto_call_parameter() - kwargs.update({'Parameters': parameters}) + kwargs.update({"Parameters": parameters}) response = self.connection_manager.call( - service="cloudformation", - command="estimate_template_cost", - kwargs=kwargs + service="cloudformation", command="estimate_template_cost", kwargs=kwargs ) self.logger.debug( "%s - Estimate Stack cost response: %s", self.stack.name, response @@ -642,10 +706,7 @@ def _format_parameters(self, parameters): continue if isinstance(value, list): value = ",".join(value) - formatted_parameters.append({ - "ParameterKey": name, - "ParameterValue": value - }) + formatted_parameters.append({"ParameterKey": name, "ParameterValue": value}) return formatted_parameters @@ -658,10 +719,8 @@ def _get_role_arn(self): :returns: The a Role ARN :rtype: dict """ - if self.stack.role_arn: - return { - "RoleARN": self.stack.role_arn - } + if self.stack.cloudformation_service_role: + return {"RoleARN": self.stack.cloudformation_service_role} else: return {} @@ -674,9 +733,7 @@ def _get_stack_timeout(self): :rtype: dict """ if self.stack.stack_timeout: - return { - "TimeoutInMinutes": self.stack.stack_timeout - } + return {"TimeoutInMinutes": self.stack.stack_timeout} else: return {} @@ -692,15 +749,17 @@ def _protect_execution(self): "currently enabled".format(self.stack.name) ) - def _wait_for_completion(self, timeout=0): + def _wait_for_completion( + self, timeout=0, boto_response: Optional[dict] = None + ) -> StackStatus: """ Waits for a Stack operation to finish. Prints CloudFormation events while it waits. :param timeout: Timeout before returning, in minutes. + :param boto_response: Response from the boto call which initiated the stack change. :returns: The final Stack status. - :rtype: sceptre.stack_status.StackStatus """ timeout = 60 * timeout @@ -709,13 +768,16 @@ def timed_out(elapsed): status = StackStatus.IN_PROGRESS - self.most_recent_event_datetime = ( - datetime.now(tzutc()) - timedelta(seconds=3) - ) + most_recent_event_datetime = extract_datetime_from_aws_response_headers( + boto_response + ) or (datetime.now(tzutc()) - timedelta(seconds=3)) + elapsed = 0 while status == StackStatus.IN_PROGRESS and not timed_out(elapsed): status = self._get_simplified_status(self._get_status()) - self._log_new_events() + most_recent_event_datetime = self._log_new_events( + most_recent_event_datetime + ) time.sleep(4) elapsed += 4 @@ -725,7 +787,7 @@ def _describe(self): return self.connection_manager.call( service="cloudformation", command="describe_stacks", - kwargs={"StackName": self.stack.external_name} + kwargs={"StackName": self.stack.external_name}, ) def _get_status(self): @@ -764,29 +826,32 @@ def _get_simplified_status(status): elif status.endswith("_FAILED"): return StackStatus.FAILED else: - raise UnknownStackStatusError( - "{0} is unknown".format(status) - ) + raise UnknownStackStatusError("{0} is unknown".format(status)) - def _log_new_events(self): + def _log_new_events(self, after_datetime: datetime) -> datetime: """ Log the latest Stack events while the Stack is being built. + + :param after_datetime: Only events after this datetime will be logged. + :returns: The datetime of the last logged event or after_datetime if no events were logged. """ events = self.describe_events()["StackEvents"] events.reverse() - new_events = [ - event for event in events - if event["Timestamp"] > self.most_recent_event_datetime - ] + new_events = [event for event in events if event["Timestamp"] > after_datetime] for event in new_events: - self.logger.info(" ".join([ - self.stack.name, - event["LogicalResourceId"], - event["ResourceType"], - event["ResourceStatus"], - event.get("ResourceStatusReason", "") - ])) - self.most_recent_event_datetime = event["Timestamp"] + self.logger.info( + " ".join( + [ + self.stack.name, + event["LogicalResourceId"], + event["ResourceType"], + event["ResourceStatus"], + event.get("ResourceStatusReason", ""), + ] + ) + ) + after_datetime = event["Timestamp"] + return after_datetime def wait_for_cs_completion(self, change_set_name): """ @@ -819,12 +884,19 @@ def _get_cs_status(self, change_set_name): cs_status = cs_description["Status"] cs_exec_status = cs_description["ExecutionStatus"] possible_statuses = [ - "CREATE_PENDING", "CREATE_IN_PROGRESS", - "CREATE_COMPLETE", "DELETE_COMPLETE", "FAILED" + "CREATE_PENDING", + "CREATE_IN_PROGRESS", + "CREATE_COMPLETE", + "DELETE_COMPLETE", + "FAILED", ] possible_execution_statuses = [ - "UNAVAILABLE", "AVAILABLE", "EXECUTE_IN_PROGRESS", - "EXECUTE_COMPLETE", "EXECUTE_FAILED", "OBSOLETE" + "UNAVAILABLE", + "AVAILABLE", + "EXECUTE_IN_PROGRESS", + "EXECUTE_COMPLETE", + "EXECUTE_FAILED", + "OBSOLETE", ] if cs_status not in possible_statuses: @@ -836,25 +908,254 @@ def _get_cs_status(self, change_set_name): "ExecutionStatus {0} is unknown".format(cs_status) ) - if ( - cs_status == "CREATE_COMPLETE" and - cs_exec_status == "AVAILABLE" - ): + if cs_status == "CREATE_COMPLETE" and cs_exec_status == "AVAILABLE": return StackChangeSetStatus.READY - elif ( - cs_status in [ - "CREATE_PENDING", "CREATE_IN_PROGRESS", "CREATE_COMPLETE" - ] and - cs_exec_status in ["UNAVAILABLE", "AVAILABLE"] - ): + elif cs_status in [ + "CREATE_PENDING", + "CREATE_IN_PROGRESS", + "CREATE_COMPLETE", + ] and cs_exec_status in ["UNAVAILABLE", "AVAILABLE"]: return StackChangeSetStatus.PENDING - elif ( - cs_status in ["DELETE_COMPLETE", "FAILED"] or - cs_exec_status in [ - "EXECUTE_IN_PROGRESS", "EXECUTE_COMPLETE", - "EXECUTE_FAILED", "OBSOLETE" - ] - ): + elif cs_status in ["DELETE_COMPLETE", "FAILED"] or cs_exec_status in [ + "EXECUTE_IN_PROGRESS", + "EXECUTE_COMPLETE", + "EXECUTE_FAILED", + "OBSOLETE", + ]: return StackChangeSetStatus.DEFUNCT else: # pragma: no cover raise Exception("This else should not be reachable.") + + def fetch_remote_template(self) -> Optional[str]: + """ + Returns the Template for the remote Stack + + :returns: the template body. + """ + self.logger.debug(f"{self.stack.name} - Fetching remote template") + + original_template = self._fetch_original_template_stage() + + if isinstance(original_template, dict): + # While not documented behavior, boto3 will attempt to deserialize the TemplateBody + # with json.loads and return the template as a dict if it is successful; otherwise (such + # as in when the template is in yaml, it will return the string. Therefore, we need to + # dump the template to json if we get a dict. + original_template = json.dumps(original_template, indent=4) + + return original_template + + def _fetch_original_template_stage(self) -> Optional[Union[str, dict]]: + try: + response = self.connection_manager.call( + service="cloudformation", + command="get_template", + kwargs={ + "StackName": self.stack.external_name, + "TemplateStage": "Original", + }, + ) + return response["TemplateBody"] + # Sometimes boto returns a string, sometimes a dictionary + except botocore.exceptions.ClientError as e: + # AWS returns a ValidationError if the stack doesn't exist + if e.response["Error"]["Code"] == "ValidationError": + return None + raise + + def fetch_remote_template_summary(self): + return self._get_template_summary(StackName=self.stack.external_name) + + def fetch_local_template_summary(self): + boto_call_parameter = self.stack.template.get_boto_call_parameter() + return self._get_template_summary(**boto_call_parameter) + + def _get_template_summary(self, **kwargs) -> Optional[dict]: + try: + template_summary = self.connection_manager.call( + service="cloudformation", command="get_template_summary", kwargs=kwargs + ) + return template_summary + except botocore.exceptions.ClientError as e: + error_response = e.response["Error"] + if ( + error_response["Code"] == "ValidationError" + and "does not exist" in error_response["Message"] + ): + return None + raise + + @add_stack_hooks + def diff(self, stack_differ: "StackDiffer") -> "StackDiff": + """ + Returns a diff of local and deployed template and stack configuration using a specific diff + library. + + :param stack_differ: The differ to use + :returns: A StackDiff object with the full, computed diff + """ + return stack_differ.diff(self) + + @add_stack_hooks + def drift_detect(self) -> Dict[str, str]: + """ + Show stack drift for a running stack. + + :returns: The stack drift detection status. + If the stack does not exist, we return a detection and + stack drift status of STACK_DOES_NOT_EXIST. + If drift detection times out after 5 minutes, we return + TIMED_OUT. + """ + try: + self._get_status() + except StackDoesNotExistError: + self.logger.info(f"{self.stack.name} - Does not exist.") + return { + "DetectionStatus": "STACK_DOES_NOT_EXIST", + "StackDriftStatus": "STACK_DOES_NOT_EXIST", + } + + response = self._detect_stack_drift() + detection_id = response["StackDriftDetectionId"] + + try: + response = self._wait_for_drift_status(detection_id) + except TimeoutError as exc: + self.logger.info(f"{self.stack.name} - {exc}") + response = {"DetectionStatus": "TIMED_OUT", "StackDriftStatus": "TIMED_OUT"} + + return response + + @add_stack_hooks + def drift_show(self, drifted: bool = False) -> Tuple[str, dict]: + """ + Detect drift status on stacks. + + :param drifted: Filter out IN_SYNC resources. + :returns: The detection status and resource drifts. + """ + response = self.drift_detect() + detection_status = response["DetectionStatus"] + + if detection_status in ["DETECTION_COMPLETE", "DETECTION_FAILED"]: + response = self._describe_stack_resource_drifts() + elif detection_status in ["TIMED_OUT", "STACK_DOES_NOT_EXIST"]: + response = {"StackResourceDriftStatus": detection_status} + else: + raise Exception("Not expected to be reachable") + + response = self._filter_drifts(response, drifted) + return (detection_status, response) + + def _wait_for_drift_status(self, detection_id: str) -> dict: + """ + Waits for drift detection to complete. + + :param detection_id: The drift detection ID. + :returns: The response from describe_stack_drift_detection_status. + """ + timeout = 300 + sleep_interval = 10 + elapsed = 0 + + while True: + if elapsed >= timeout: + raise TimeoutError(f"Timed out after {elapsed} seconds") + + self.logger.info(f"{self.stack.name} - Waiting for drift detection") + response = self._describe_stack_drift_detection_status(detection_id) + detection_status = response["DetectionStatus"] + + self._log_drift_status(response) + + if detection_status == "DETECTION_IN_PROGRESS": + time.sleep(sleep_interval) + elapsed += sleep_interval + else: + return response + + def _log_drift_status(self, response: dict) -> None: + """ + Log the drift status while waiting for + drift detection to complete. + """ + keys = [ + "StackDriftDetectionId", + "DetectionStatus", + "DetectionStatusReason", + "StackDriftStatus", + ] + + for key in keys: + if key in response: + self.logger.debug(f"{self.stack.name} - {key} - {response[key]}") + + def _detect_stack_drift(self) -> dict: + """ + Run detect_stack_drift. + """ + self.logger.info(f"{self.stack.name} - Detecting Stack Drift") + + return self.connection_manager.call( + service="cloudformation", + command="detect_stack_drift", + kwargs={"StackName": self.stack.external_name}, + ) + + def _describe_stack_drift_detection_status(self, detection_id: str) -> dict: + """ + Run describe_stack_drift_detection_status. + """ + self.logger.info(f"{self.stack.name} - Describing Stack Drift Detection Status") + + return self.connection_manager.call( + service="cloudformation", + command="describe_stack_drift_detection_status", + kwargs={"StackDriftDetectionId": detection_id}, + ) + + def _describe_stack_resource_drifts(self) -> dict: + """ + Detects stack resource_drifts for a running stack. + """ + self.logger.info(f"{self.stack.name} - Describing Stack Resource Drifts") + + return self.connection_manager.call( + service="cloudformation", + command="describe_stack_resource_drifts", + kwargs={"StackName": self.stack.external_name}, + ) + + def _filter_drifts(self, response: dict, drifted: bool) -> dict: + """ + The filtered response after filtering out StackResourceDriftStatus. + :param drifted: Filter out IN_SYNC resources from CLI --drifted. + """ + if "StackResourceDrifts" not in response: + return response + + result = {"StackResourceDrifts": []} + include_all_drift_statuses = not drifted + + for drift in response["StackResourceDrifts"]: + is_drifted = drift["StackResourceDriftStatus"] != "IN_SYNC" + if include_all_drift_statuses or is_drifted: + result["StackResourceDrifts"].append(drift) + + return result + + @add_stack_hooks + def dump_config(self, config_reader: ConfigReader): + """ + Dump the config for a stack. + """ + stack_path = normalise_path(self.stack.name + ".yaml") + return config_reader.read(stack_path) + + @add_stack_hooks + def dump_template(self): + """ + Returns the Template for the Stack + """ + return self.stack.template.body diff --git a/sceptre/plan/executor.py b/sceptre/plan/executor.py index a975536cb..501edf14c 100644 --- a/sceptre/plan/executor.py +++ b/sceptre/plan/executor.py @@ -7,49 +7,46 @@ executing the command specified in a SceptrePlan. """ import logging - from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Set + from sceptre.plan.actions import StackActions -from sceptre.stack_status import StackStatus +from sceptre.stack import Stack class SceptrePlanExecutor(object): - - def __init__(self, command, launch_order): + def __init__(self, command: str, launch_order: List[Set[Stack]]): """ - Initalises a SceptrePlanExecutor, generates the launch order, threads + Initialises a SceptrePlanExecutor, generates the launch order, threads and intial Stack Statuses. :param command: The command to execute on the Stack. - :type command: str - :param launch_order: A list containing sets of Stacks that can be\ - executed concurrently. - :type launch_order: list + :param launch_order: A list containing sets of Stacks that can be executed concurrently. """ self.logger = logging.getLogger(__name__) self.command = command self.launch_order = launch_order - - self.num_threads = len(max(launch_order, key=len)) - self.stack_statuses = {stack: StackStatus.PENDING - for batch in launch_order for stack in batch} + # Select the number of threads based upon the max batch size, + # or use 1 if all batches are empty + self.num_threads = len(max(launch_order, key=len)) or 1 def execute(self, *args): """ - Execute is responsible executing the sets of Stacks in `launch_order` + Execute is responsible executing the sets of Stacks in launch_order concurrently, in the correct order. - :param \*args: Any arguments that should be passed through to the\ + :param args: Any arguments that should be passed through to the StackAction being called. """ responses = {} with ThreadPoolExecutor(max_workers=self.num_threads) as executor: for batch in self.launch_order: - futures = [executor.submit(self._execute, stack, *args) - for stack in batch] + futures = [ + executor.submit(self._execute, stack, *args) for stack in batch + ] for future in as_completed(futures): stack, status = future.result() diff --git a/sceptre/plan/plan.py b/sceptre/plan/plan.py index 22e436002..5da40c6f5 100644 --- a/sceptre/plan/plan.py +++ b/sceptre/plan/plan.py @@ -6,46 +6,59 @@ This module implements a SceptrePlan, which is responsible for holding all nessessary information for a command to execute. """ +import functools +import itertools from os import path, walk +from typing import Dict, List, Set, Callable, Iterable, Optional -from sceptre.exceptions import ConfigFileNotFoundError from sceptre.config.graph import StackGraph from sceptre.config.reader import ConfigReader -from sceptre.plan.executor import SceptrePlanExecutor +from sceptre.context import SceptreContext +from sceptre.diffing.stack_differ import StackDiff +from sceptre.exceptions import ConfigFileNotFoundError from sceptre.helpers import sceptreise_path +from sceptre.plan.executor import SceptrePlanExecutor +from sceptre.stack import Stack -class SceptrePlan(object): +def require_resolved(func) -> Callable: + @functools.wraps(func) + def wrapped(self: "SceptrePlan", *args, **kwargs): + if self.launch_order is None: + raise RuntimeError(f"You cannot call {func.__name__}() before resolve().") + return func(self, *args, **kwargs) + + return wrapped + - def __init__(self, context): +class SceptrePlan(object): + def __init__(self, context: SceptreContext): """ Intialises a SceptrePlan and generates the Stacks, StackGraph and launch order of required. :param context: A SceptreContext - :type sceptre.context.SceptreContext: """ self.context = context self.command = None self.reverse = None - self.launch_order = None + self.launch_order: Optional[List[Set[Stack]]] = None - config_reader = ConfigReader(context) - all_stacks, command_stacks = config_reader.construct_stacks() + self.config_reader = ConfigReader(context) + all_stacks, command_stacks = self.config_reader.construct_stacks() self.graph = StackGraph(all_stacks) self.command_stacks = command_stacks + @require_resolved def _execute(self, *args): executor = SceptrePlanExecutor(self.command, self.launch_order) return executor.execute(*args) - def _generate_launch_order(self, reverse=False): + def _generate_launch_order(self, reverse=False) -> List[Set[Stack]]: if self.context.ignore_dependencies: return [self.command_stacks] graph = self.graph.filtered(self.command_stacks, reverse) - if self.context.ignore_dependencies: - return [self.command_stacks] launch_order = [] while graph.graph: @@ -60,12 +73,39 @@ def _generate_launch_order(self, reverse=False): if not launch_order: raise ConfigFileNotFoundError( - "No stacks detected from the given path '{}'. Valid stack paths are: {}" - .format(sceptreise_path(self.context.command_path), self._valid_stack_paths()) + "No stacks detected from the given path '{}'. Valid stack paths are: {}".format( + sceptreise_path(self.context.command_path), + self._valid_stack_paths(), + ) ) return launch_order + @require_resolved + def __iter__(self) -> Iterable[Stack]: + """Iterates the stacks in the launch_order""" + # We cast it to list so it's "frozen" in time, in case the launch order is modified + # while iterating. + yield from list(itertools.chain.from_iterable(self.launch_order)) + + @require_resolved + def remove_stack_from_plan(self, stack: Stack): + for batch in self.launch_order: + if stack in batch: + batch.remove(stack) + return + + @require_resolved + def filter(self, predicate: Callable[[Stack], bool]): + """Filters the plan's resolved launch_order to remove specific stacks. + + :param predicate: This callable should take a single Stack and return True if it should stay + in the launch_order or False if it should be filtered out. + """ + for stack in self: + if not predicate(stack): + self.remove_stack_from_plan(stack) + def resolve(self, command, reverse=False): if command == self.command and reverse == self.reverse: return @@ -206,7 +246,7 @@ def continue_update_rollback(self, *args): :returns: A dictionary of Stacks :rtype: dict - """ + """ self.resolve(command=self.continue_update_rollback.__name__) return self._execute(*args) @@ -349,8 +389,64 @@ def generate(self, *args): def _valid_stack_paths(self): return [ - sceptreise_path(path.relpath(path.join(dirpath, f), self.context.config_path)) + sceptreise_path( + path.relpath(path.join(dirpath, f), self.context.config_path) + ) for dirpath, dirnames, files in walk(self.context.config_path) for f in files if not f.endswith(self.context.config_file) ] + + def fetch_remote_template(self, *args): + """ + Returns a generated Template for a given Stack + + :returns: A list of Stacks and their template body. + :rtype: List[str] + """ + self.resolve(command=self.fetch_remote_template.__name__) + return self._execute(*args) + + def diff(self, *args) -> Dict[Stack, StackDiff]: + """ + Show diffs between the running and generated stack. + + :returns: A dict where the keys are Stack objects and the values are StackDiffs. + """ + self.resolve(command=self.diff.__name__) + return self._execute(*args) + + def drift_detect(self, *args) -> Dict[Stack, str]: + """ + Show drift detection status of a stack. + + :returns: A list of detected drift against running stacks. + """ + self.resolve(command=self.drift_detect.__name__) + return self._execute(*args) + + def drift_show(self, *args) -> Dict[Stack, str]: + """ + Show stack drift for a running stack. + + :returns: A list of detected drift against running stacks. + """ + self.resolve(command=self.drift_show.__name__) + return self._execute(*args) + + def dump_config(self, *args): + """ + Dump the config for a stack. + """ + self.resolve(command=self.dump_config.__name__) + return self._execute(self.config_reader, *args) + + def dump_template(self, *args): + """ + Returns a generated Template for a given Stack + + :returns: A dictionary of Stacks and their template body. + :rtype: dict + """ + self.resolve(command=self.generate.__name__) + return self._execute(*args) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 846ee2076..06fd34f7f 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -3,42 +3,185 @@ import logging from contextlib import contextmanager from threading import RLock +from typing import Any, TYPE_CHECKING, Type, Union, TypeVar -import six -from sceptre.helpers import _call_func_on_values +from sceptre.exceptions import InvalidResolverArgumentError +from sceptre.helpers import _call_func_on_values, delete_keys_from_containers +from sceptre.logging import StackLoggerAdapter +from sceptre.resolvers.placeholders import ( + create_placeholder_value, + are_placeholders_enabled, + PlaceholderType, +) +if TYPE_CHECKING: + from sceptre import stack -class RecursiveGet(Exception): +T_Container = TypeVar("T_Container", bound=Union[dict, list]) +Self = TypeVar("Self") + + +class RecursiveResolve(Exception): pass -@six.add_metaclass(abc.ABCMeta) -class Resolver: - """ - Resolver is an abstract base class that should be inherited by all - Resolvers. +class CustomYamlTagBase: + """A base class for custom Yaml Elements (i.e. hooks and resolvers). + + This base class takes care of common functionality needed by subclasses: + | * logging setup (associated with stacks) + | * Creating unique clones associated with individual stacks (applied recursively down to all + | resolvers in the argument) + | * On-stack-connect setup that might be needed once connected to a Stack (applied recursively down to + | all resolvers in the argument) + | * Automatically resolving resolvers in the argument when accessing self.argument - :param argument: The argument of the resolver. - :type argument: str - :param stack: The associated stack of the resolver. - :type stack: sceptre.stack.Stack """ - __metaclass__ = abc.ABCMeta + logger = logging.getLogger(__name__) + + def __init__(self, argument: Any = None, stack: "stack.Stack" = None): + """Initializes a custom yaml tag object. + + :param argument: The argument passed to the yaml tag. This could be a string, number, list, + or dict. Resolvers are supported in the argument, but they must be in either list or dict + arguments. + :param stack: The stack object associated with this instance. NOTE: When first instantiated + from loading the YAML, there will be no Stack. A Stack instance will only be passed into + the instance when "clone_for_stack" is invoked when the tag is associated with a specific + stack. + """ + if stack is not None: + self.logger = StackLoggerAdapter(self.logger, stack.name) - def __init__(self, argument=None, stack=None): - self.logger = logging.getLogger(__name__) - self.argument = argument self.stack = stack + self._argument = argument + self._argument_is_resolved = False + + @property + def argument(self) -> Any: + """This is the resolver or hook's argument. + + This property will resolve all nested resolvers inside the argument, but only if this + instance has been associated with a Stack. + + Resolving nested resolvers will result in their values being replaced in the dict/list they + were in with their resolved value, so we won't have to resolve them again. + + Any resolvers that "resolve to nothing" (i.e. return None) will be removed from the dict/list + they were in. + + If this property is accessed BEFORE the instance has a stack, it will return + the raw argument value. This is to safeguard any __init__() behaviors from triggering + resolution prematurely. + """ + if self.stack is not None and not self._argument_is_resolved: + # Since resolving the argument updates the argument list or dict so that there aren't + # resolvers in it any more, we only need to do this resolution once per instance. + self._resolve_argument() + + return self._argument + + @argument.setter + def argument(self, value): + self._argument = value + + def _resolve_argument(self): + """Resolves all argument resolvers recursively.""" + + keys_to_delete = [] + + def resolve(containing_list_or_dict, key, obj: Resolver): + result = obj.resolve() + # If the resolver "resolves to nothing", then it should get deleted out of its container. + if result is None: + keys_to_delete.append((containing_list_or_dict, key)) + else: + containing_list_or_dict[key] = result + + _call_func_on_values(resolve, self._argument, Resolver) + delete_keys_from_containers(keys_to_delete) + + self._argument_is_resolved = True + + def _recursively_setup(self): + """Ensures all nested resolvers in this resolver's argument are also setup when this + instance's setup method is called. + """ + self.setup() + + def setup_nested(containing_list_or_dict, key, obj: Resolver): + obj._recursively_setup() + + _call_func_on_values(setup_nested, self._argument, Resolver) + + def _recursively_clone(self: Self, stack: "stack.Stack") -> Self: + """Recursively clones the instance and its arguments. + + The returned instance will have an identical argument that is a different memory reference, + so that instances inherited from a stack group and applied across multiple stacks are + independent of each other. + + Furthermore, all nested resolvers in this resolver's argument will also be cloned to ensure + they themselves are also independent and fully configured for the current stack. + """ + + def recursively_clone_arguments(obj): + if isinstance(obj, Resolver): + return obj._recursively_clone(stack) + if isinstance(obj, list): + return [recursively_clone_arguments(item) for item in obj] + elif isinstance(obj, dict): + return { + key: recursively_clone_arguments(val) for key, val in obj.items() + } + return obj + + argument = recursively_clone_arguments(self._argument) + clone = type(self)(argument, stack) + return clone + + def clone_for_stack(self: Self, stack: "stack.Stack") -> Self: + """ + Obtains a clone of the current object, setup and ready for use for a given Stack instance. + """ + clone = self._recursively_clone(stack) + clone._recursively_setup() + return clone + def setup(self): """ - This method is called at during stack initialisation. + This method is called when the object is connected to a Stack instance. Implementation of this method in subclasses can be used to do any initial setup of the object. """ pass # pragma: no cover + def __repr__(self) -> str: + """Returns a string representation of the resolver. + + This is mostly used for resolver placeholders. In cases where we cannot resolve a resolver + YET, such as when we're generating a template or diff and there's a dependency on a stack + output of a stack that hasn't been deployed yet, placeholders need to render the resolver + in a way that is useful. + + We use self._argument instead of self.argument because if there are resolvers nested in this + resolver's argument and one of those cannot be resolved, we'll need those resolvers to also + be converted to useful placeholder values. + """ + as_str = f"!{self.__class__.__name__}" + if self._argument is not None: + as_str += f"({self._argument})" + + return as_str + + +class Resolver(CustomYamlTagBase, metaclass=abc.ABCMeta): + """ + Resolver is an abstract base class that should be subclassed by all Resolvers. + """ + @abc.abstractmethod def resolve(self): """ @@ -49,62 +192,246 @@ def resolve(self): """ pass # pragma: no cover + def raise_invalid_argument_error(self, message, from_: Exception = None): + error_message = f"{self.stack.name} - {message}" + if from_: + raise InvalidResolverArgumentError(error_message) from from_ + raise InvalidResolverArgumentError(error_message) + -class ResolvableProperty(object): +class ResolvableProperty(abc.ABC): """ - This is a descriptor class used to store an attribute that may contain - Resolver objects. When retrieving the dictionary or list, any Resolver - objects contains are a value or within a list are resolved to a primitive - type. Supports nested dictionary and lists. + This is an abstract base class for a descriptor used to store an attribute that have values + associated with Resolver objects. :param name: Attribute suffix used to store the property in the instance. - :type name: str + :param placeholder_type: The type of placeholder that should be returned, when placeholders are + allowed, when a resolver can't be resolved. """ - def __init__(self, name): + def __init__(self, name: str, placeholder_type=PlaceholderType.explicit): self.name = "_" + name self.logger = logging.getLogger(__name__) - self._get_in_progress = False + self.placeholder_type = placeholder_type + self._lock = RLock() - def __get__(self, instance, type): + def __get__(self, stack: "stack.Stack", stack_class: Type["stack.Stack"]) -> Any: """ - Attribute getter which resolves any Resolver object contained in the - complex data structure. + Attribute getter which resolves the resolver(s). + :param stack: The Stack instance the property is being retrieved for + :param stack_class: The class of the stack that the property is being retrieved for. :return: The attribute stored with the suffix ``name`` in the instance. - :rtype: dict or list - """ - with self._lock, self._no_recursive_get(): - def resolve(attr, key, value): - try: - attr[key] = value.resolve() - except RecursiveGet: - attr[key] = self.ResolveLater(instance, self.name, key, - lambda: value.resolve()) - - if hasattr(instance, self.name): - retval = _call_func_on_values( - resolve, getattr(instance, self.name), Resolver - ) - return retval + :rtype: The obtained value, as resolved by the property + """ + with self._lock, self._no_recursive_get(stack): + if hasattr(stack, self.name): + return self.get_resolved_value(stack, stack_class) - def __set__(self, instance, value): + def __set__(self, stack: "stack.Stack", value: Any): """ Attribute setter which adds a stack reference to any resolvers in the data structure `value` and calls the setup method. + :param stack: The Stack instance the value is being set onto + :param value: The value being set on the property + """ + with self._lock: + self.assign_value_to_stack(stack, value) + + @contextmanager + def _no_recursive_get(self, stack: "stack.Stack"): + # We don't care about recursive gets on the same property but different Stack instances, + # only recursive gets on the same stack. Some Resolvers access the same property on OTHER + # stacks and that actually shouldn't be a problem. Remember, these descriptor instances are + # set on the CLASS and so instance variables on them are shared across all classes that + # access them. Thus, we set this "get_in_progress" attribute on the stack instance rather + # than the descriptor instance. + get_status_name = f"_{self.name}_get_in_progress" + if getattr(stack, get_status_name, False): + raise RecursiveResolve( + f"Resolving Stack.{self.name[1:]} required resolving itself" + ) + setattr(stack, get_status_name, True) + try: + yield + finally: + setattr(stack, get_status_name, False) + + @abc.abstractmethod + def get_resolved_value( + self, stack: "stack.Stack", stack_class: Type["stack.Stack"] + ) -> Any: + """Implement this method to return the value of the resolvable_property.""" + pass + + @abc.abstractmethod + def assign_value_to_stack(self, stack: "stack.Stack", value: Any): + """Implement this method to assign the value to the resolvable property.""" + pass + + def resolve_resolver_value(self, resolver: "Resolver") -> Any: + """Returns the resolved parameter value. + + If the resolver happens to raise an error and placeholders are currently allowed for resolvers, + a placeholder will be returned instead of reraising the error. + + :param resolver: The resolver to resolve. + :return: The resolved value (or placeholder, in certain circumstances) """ - def setup(attr, key, value): - value.stack = instance - value.setup() + try: + return resolver.resolve() + except RecursiveResolve: + # Recursive resolve issues shouldn't be masked by a placeholder. + raise + except Exception: + if are_placeholders_enabled(): + placeholder_value = create_placeholder_value( + resolver, self.placeholder_type + ) + + self.logger.debug( + "Error encountered while resolving the resolver. This is allowed for the current " + f"operation. Resolving it to a placeholder value instead: {placeholder_value}" + ) + return placeholder_value + raise + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}({self.name[1:]})>" + + +class ResolvableContainerProperty(ResolvableProperty): + """ + This is a descriptor class used to store an attribute that may CONTAIN + Resolver objects. Resolvers will be resolved upon access of this property. + When resolvers are resolved, they will be replaced in the container with their + resolved value, in order to avoid redundant resolutions. + + Supports nested dictionary and lists. + + :param name: Attribute suffix used to store the property in the instance. + :type name: str + """ + + def __get__( + self, stack: "stack.Stack", stack_class: Type["stack.Stack"] + ) -> T_Container: + container = super().__get__(stack, stack_class) with self._lock: - _call_func_on_values(setup, value, Resolver) - setattr(instance, self.name, value) + # Resolve any deferred resolvers, now that the recursive get lock has been released. + self._resolve_deferred_resolvers(stack, container) + + return container + + def get_resolved_value( + self, stack: "stack.Stack", stack_class: Type["stack.Stack"] + ) -> T_Container: + """Obtains the resolved value for this property. Any resolvers that resolve to None will have + their key/index removed from their dict/list where they are. Other resolvers will have their + key/index's value replace with the resolved value to avoid redundant resolutions. + + :param stack: The Stack instance to obtain the value for + :param stack_class: The class of the Stack instance. + :return: The fully resolved container. + """ + keys_to_delete = [] + + def resolve(attr: Union[dict, list], key: Union[int, str], value: Resolver): + # Update the container key's value with the resolved value, if possible... + try: + result = self.resolve_resolver_value(value) + if result is None: + self.logger.debug( + f"Removing item {key} because resolver returned None." + ) + # We gather up resolvers (and their immediate containers) that resolve to None, + # since that really means the resolver resolves to nothing. This is not common, + # but should be supported. We gather these rather than immediately remove them + # because this function is called in the context of looping over that attr, so + # we cannot alter its size until after the loop is complete. + keys_to_delete.append((attr, key)) + else: + attr[key] = result + except RecursiveResolve: + # It's possible that resolving the resolver might attempt to access another + # resolvable property's value in this same container. In this case, we'll delay + # resolution and instead return a ResolveLater so the value can be resolved outside + # this recursion. + attr[key] = self.ResolveLater( + stack, + self.name, + key, + lambda: value.resolve(), + ) + + container = getattr(stack, self.name) + _call_func_on_values(resolve, container, Resolver) + delete_keys_from_containers(keys_to_delete) + + return container + + def assign_value_to_stack(self, stack: "stack.Stack", value: Union[dict, list]): + """Assigns a COPY of the specified value to the stack instance. This method copies the value + rather than directly assigns it to avoid bugs related to shared objects in memory. - class ResolveLater(object): + :param stack: The stack to assign the value to + :param value: The value to assign + """ + cloned = self._clone_container_with_resolvers(value, stack) + setattr(stack, self.name, cloned) + + def _clone_container_with_resolvers( + self, container: T_Container, stack: "stack.Stack" + ) -> T_Container: + """Recurses into the container, cloning and setting up resolvers and creating a copy of all + nested containers. + + :param container: The container being recursed into and cloned + :param stack: The stack the container is being copied for + :return: The fully copied container with resolvers fully set up. + """ + + def recurse(obj): + if isinstance(obj, Resolver): + return obj.clone_for_stack(stack) + if isinstance(obj, list): + return [recurse(item) for item in obj] + elif isinstance(obj, dict): + return {key: recurse(val) for key, val in obj.items()} + return obj + + return recurse(container) + + def _resolve_deferred_resolvers(self, stack: "stack.Stack", container: T_Container): + def raise_if_not_resolved(attr, key, value): + # If this function has been hit, it means that after attempting to resolve all the + # ResolveLaters, there STILL are ResolveLaters left in the container. Rather than + # continuing to try to resolve (possibly infinitely), we'll raise a RecursiveGet to + # break that infinite loop. This situation would happen if a resolver accesses a resolver + # in the same container, which then accesses another resolver (possibly the same one) in + # the same container. + raise RecursiveResolve( + f"Resolving Stack.{self.name[1:]} required resolving itself" + ) + + has_been_resolved_attr_name = f"{self.name}_is_resolved" + if not getattr(stack, has_been_resolved_attr_name, False): + # We set it first rather than after to avoid entering this block again on this property + # for this stack. + setattr(stack, has_been_resolved_attr_name, True) + _call_func_on_values( + lambda attr, key, value: value(), container, self.ResolveLater + ) + # Search the container to see if there are any ResolveLaters left; + # Raise a RecursiveResolve if there are. + _call_func_on_values(raise_if_not_resolved, container, self.ResolveLater) + + class ResolveLater: """Represents a value that could not yet be resolved but can be resolved in the future.""" + def __init__(self, instance, name, key, resolution_function): self._instance = instance self._name = name @@ -114,14 +441,55 @@ def __init__(self, instance, name, key, resolution_function): def __call__(self): """Resolve the value.""" attr = getattr(self._instance, self._name) - attr[self._key] = self._resolution_function() + result = self._resolution_function() + if result is None: + self.logger.debug( + f"Removing item {self._key} because resolver returned None." + ) + del attr[self._key] + else: + attr[self._key] = result - @contextmanager - def _no_recursive_get(self): - if self._get_in_progress: - raise RecursiveGet() - self._get_in_progress = True - try: - yield - finally: - self._get_in_progress = False + +class ResolvableValueProperty(ResolvableProperty): + """ + This is a descriptor class used to store an attribute that may BE a single + Resolver object. If it is a resolver, it will be resolved upon access of this property. + When resolved, the resolved value will replace the resolver on the stack in order to avoid + redundant resolutions. + + :param name: Attribute suffix used to store the property in the instance. + :type name: str + """ + + def get_resolved_value( + self, stack: "stack.Stack", stack_class: Type["stack.Stack"] + ) -> Any: + """Gets the fully-resolved value from the property. Resolvers will be replaced on the stack + instance with their resolved value to avoid redundant resolutions. + + :param stack: The Stack instance to obtain the value from + :param stack_class: The class of the Stack instance + :return: The fully resolved value + """ + raw_value = getattr(stack, self.name) + if isinstance(raw_value, Resolver): + value = self.resolve_resolver_value(raw_value) + # Overwrite the stored resolver value with the resolved value to avoid resolving the + # same value multiple times. + setattr(stack, self.name, value) + else: + value = raw_value + + return value + + def assign_value_to_stack(self, stack: "stack.Stack", value: Any): + """Assigns the value to the Stack instance passed, setting up and cloning the value if it + is a Resolver. + + :param stack: The Stack instance to set the value on + :param value: The value to set + """ + if isinstance(value, Resolver): + value = value.clone_for_stack(stack) + setattr(stack, self.name, value) diff --git a/sceptre/resolvers/join.py b/sceptre/resolvers/join.py new file mode 100644 index 000000000..3910d3327 --- /dev/null +++ b/sceptre/resolvers/join.py @@ -0,0 +1,36 @@ +from sceptre.resolvers import Resolver + + +class Join(Resolver): + """This resolver allows you to join multiple strings together to form a single string. This is + great for combining the outputs of multiple resolvers. This resolver works just like + CloudFormation's ``!Join`` intrinsic function. + + The argument for this resolver should be a list with two elements: (1) A string to join the + elements on and (2) a list of items to join. + + Example: + + parameters: + BaseUrl: !join + - ":" + - - !stack_output my/app/stack.yaml::HostName + - !stack_output my/other/stack.yaml::Port + + """ + + def resolve(self): + error_message = ( + "The argument to !join must be a 2-element list, where the first element is the join " + "string and the second is a list of items to join." + ) + if not isinstance(self.argument, list) or len(self.argument) != 2: + self.raise_invalid_argument_error(error_message) + + delimiter, items_list = self.argument + if not isinstance(delimiter, str) or not isinstance(items_list, list): + self.raise_invalid_argument_error(error_message) + + string_items = map(str, items_list) + joined = delimiter.join(string_items) + return joined diff --git a/sceptre/resolvers/no_value.py b/sceptre/resolvers/no_value.py new file mode 100644 index 000000000..224f74c7b --- /dev/null +++ b/sceptre/resolvers/no_value.py @@ -0,0 +1,14 @@ +from sceptre.resolvers import Resolver + + +class NoValue(Resolver): + """This resolver resolves to nothing, functioning just like the AWS::NoValue special value. When + assigned to a resolvable Stack property, it will remove the config key/value from the stack or + the container on the stack where it has been assigned, as if this value wasn't assigned at all. + + This is mostly useful for simplifying conditional logic on Stack and StackGroup config files + where, if a certain condition is met, a value is passed, otherwise it's not passed at all. + """ + + def resolve(self) -> None: + return None diff --git a/sceptre/resolvers/placeholders.py b/sceptre/resolvers/placeholders.py new file mode 100644 index 000000000..8a08399e7 --- /dev/null +++ b/sceptre/resolvers/placeholders.py @@ -0,0 +1,96 @@ +from contextlib import contextmanager +from enum import Enum +from threading import Lock +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from sceptre import resolvers + +# This is a toggle used for globally enabling placeholder values out of resolvers when they error +# while resolving. This is important when performing actions on stacks like validation or generation +# when their dependencies have not been deployed yet and those dependencies are expressed in stack +# resolvers that are used in those actions, especially sceptre_user_data. +_RESOLVE_PLACEHOLDER_ON_ERROR = False + + +class PlaceholderType(Enum): + explicit = 1 # Looks like "{ !MyClass(argument) }" + alphanum = 2 # Looks like MyClassargument + none = 3 # Resolves to None + + +_placeholder_lock = Lock() + + +@contextmanager +def use_resolver_placeholders_on_error(): + """A context manager that toggles on placeholders for resolvers that error out. This should NOT + be used while creating/launching stacks, but it is often required when validating or diffing + stacks whose dependencies haven't yet been deployed and that reference those dependencies with + resolvers, especially in the sceptre_user_data. + """ + global _RESOLVE_PLACEHOLDER_ON_ERROR + + try: + with _placeholder_lock: + _RESOLVE_PLACEHOLDER_ON_ERROR = True + yield + finally: + with _placeholder_lock: + _RESOLVE_PLACEHOLDER_ON_ERROR = False + + +def are_placeholders_enabled() -> bool: + """Indicates whether placeholders have been globally enabled or not.""" + with _placeholder_lock: + return _RESOLVE_PLACEHOLDER_ON_ERROR + + +def create_placeholder_value( + resolver: "resolvers.Resolver", placeholder_type: PlaceholderType +) -> Any: + placeholder_func = _placeholders[placeholder_type] + return placeholder_func(resolver) + + +def _create_explicit_resolver_placeholder(resolver: "resolvers.Resolver") -> str: + """Creates a placeholder value to be substituted for the resolved value when placeholders are + allowed and the value cannot be resolved. + + The placeholder will look like one of: + * { !ClassName } -> used when there is no argument + * { !ClassName(argument) } -> used when there is a string argument + * { !ClassName({'key': 'value'}) } -> used when there is a dict argument + + :param resolver: The resolver to create a placeholder for + :return: The placeholder value + """ + return f"{{ {resolver} }}" + + +def _create_alphanumeric_placeholder(resolver: "resolvers.Resolver") -> str: + """Creates a placeholder value that is only composed of alphanumeric characters. This is more + useful when performing operations that send a template to CloudFormation, which will have stricter + requirements for values in templates. + + Values from this function will not be as readable, but they are more likely to be valid when passed + to a template. + + The placeholder will look like one of: + * ClassName -> used when there is no argument + * ClassNameargument -> used when there is a string argument + * ClassNamekeyvalue -> used when there is a dict argument + + :param resolver: The resolver to create a placeholder for + :return: The placeholder value + """ + explicit_placeholder = _create_explicit_resolver_placeholder(resolver) + alphanum_placeholder = "".join(c for c in explicit_placeholder if c.isalnum()) + return alphanum_placeholder + + +_placeholders = { + PlaceholderType.explicit: _create_explicit_resolver_placeholder, + PlaceholderType.alphanum: _create_alphanumeric_placeholder, + PlaceholderType.none: lambda resolver: None, +} diff --git a/sceptre/resolvers/select.py b/sceptre/resolvers/select.py new file mode 100644 index 000000000..92e5f94d1 --- /dev/null +++ b/sceptre/resolvers/select.py @@ -0,0 +1,42 @@ +from sceptre.resolvers import Resolver + + +class Select(Resolver): + """This resolver allows you to select a specific index from a list of items or a specific key + from a dict.. This is great for combining with the ``!split`` resolver to obtain part of a + string. This function works almost the same as CloudFormation's ``!Select`` intrinsic function, + **except (1) you can use this with negative indexes to select with a reverse index** and (2) + you can select keys from a dict. + + The argument for this resolver should be a list with two elements: (1) A numerical index or + string key and (2) a list or dict of items to select out of. If the index is negative, + it will select from the end of the list. For example, "-1" would select the last element and + "-2" would select the second-to-last element. + + Example: + + parameters: + # This selects the last element after you split the connection string on "/" + DatabaseName: !select + - -1 + - !split ["/", !stack_output my/database/stack.yaml::ConnectionString] + """ + + def resolve(self): + error_message = ( + "The argument to !select must be a two-element list, where the first element is the " + "index or key to select with and the second element is the list or dict to select from." + ) + if not isinstance(self.argument, list) or len(self.argument) != 2: + self.raise_invalid_argument_error(error_message) + + index, items = self.argument + if not isinstance(items, (dict, list)): + self.raise_invalid_argument_error(error_message) + + try: + return items[index] + except (TypeError, KeyError, IndexError) as e: + self.raise_invalid_argument_error( + f"Could not select with index/key {index}: {e}", e + ) diff --git a/sceptre/resolvers/split.py b/sceptre/resolvers/split.py new file mode 100644 index 000000000..e88a0e380 --- /dev/null +++ b/sceptre/resolvers/split.py @@ -0,0 +1,33 @@ +from sceptre.resolvers import Resolver + + +class Split(Resolver): + """This resolver will split a value on a given delimiter string. This is great when combining with the + ``!select`` resolver. This function works the same as CloudFormation's ``!Split`` intrinsic function. + + Note: The return value of this resolver is a *list*, not a string. This will not work to set Stack + configurations that expect strings, but it WILL work to set Stack configurations that expect lists. + + The argument for this resolver should be a list with two elements: (1) The delimiter to split on and + (2) a string to split. + + Example: + notifications: !split + - ";" + - !stack_output my/sns/topics.yaml::SemicolonDelimitedArns + """ + + def resolve(self): + error_message = ( + "The argument to !split must be a two-element list, where the first element is the " + "string to split on and the second element string to split." + ) + if ( + not isinstance(self.argument, list) + or len(self.argument) != 2 + or not all(isinstance(a, str) for a in self.argument) + ): + self.raise_invalid_argument_error(error_message) + + split_on, split_string = self.argument + return split_string.split(split_on) diff --git a/sceptre/resolvers/stack_attr.py b/sceptre/resolvers/stack_attr.py new file mode 100644 index 000000000..d620e6aae --- /dev/null +++ b/sceptre/resolvers/stack_attr.py @@ -0,0 +1,62 @@ +from typing import Any, List + +from sceptre.resolvers import Resolver + + +class StackAttr(Resolver): + """Resolves to the value of another field on the Stack Config, including other resolvers. + + The argument for this resolver should be the "key path" from the stack object, which can access + nested keys/indexes using a "." to separate segments. + + For example, given this Stack Config structure... + + sceptre_user_data: + nested_list: + - first + - second + + Using "!stack_attr sceptre_user_data.nested_list.1" on your stack would resolve to "second". + """ + + # These are all the attributes on Stack Configs whose names are changed when they are assigned + # to the Stack instance. + STACK_ATTR_MAP = { + "template": "template_handler_config", + "protect": "protected", + "stack_name": "external_name", + "stack_tags": "tags", + } + + def resolve(self) -> Any: + """Returns the resolved value of the field referenced by the resolver's argument.""" + segments = self.argument.split(".") + + # Remap top-level attributes to match stack config + first_segment = segments[0] + segments[0] = self.STACK_ATTR_MAP.get(first_segment, first_segment) + + if self._key_is_from_stack_group_config(first_segment): + obj = self.stack.stack_group_config + else: + obj = self.stack + + result = self._recursively_resolve_segments(obj, segments) + return result + + def _key_is_from_stack_group_config(self, key: str): + return key in self.stack.stack_group_config and not hasattr(self.stack, key) + + def _recursively_resolve_segments(self, obj: Any, segments: List[str]): + if not segments: + return obj + + attr_name, *rest = segments + if isinstance(obj, dict): + value = obj[attr_name] + elif isinstance(obj, list): + value = obj[int(attr_name)] + else: + value = getattr(obj, attr_name) + + return self._recursively_resolve_segments(value, rest) diff --git a/sceptre/resolvers/stack_output.py b/sceptre/resolvers/stack_output.py index 9ed350787..6b2254157 100644 --- a/sceptre/resolvers/stack_output.py +++ b/sceptre/resolvers/stack_output.py @@ -1,21 +1,18 @@ # -*- coding: utf-8 -*- -import abc -import six +import functools import logging import shlex from botocore.exceptions import ClientError +from sceptre.exceptions import DependencyStackMissingOutputError, StackDoesNotExistError from sceptre.helpers import normalise_path, sceptreise_path from sceptre.resolvers import Resolver -from sceptre.exceptions import DependencyStackMissingOutputError -from sceptre.exceptions import StackDoesNotExistError TEMPLATE_EXTENSION = ".yaml" -@six.add_metaclass(abc.ABCMeta) class StackOutputBase(Resolver): """ A abstract base class which provides methods for getting Stack outputs. @@ -25,7 +22,9 @@ def __init__(self, *args, **kwargs): self.logger = logging.getLogger(__name__) super(StackOutputBase, self).__init__(*args, **kwargs) - def _get_output_value(self, stack_name, output_key, profile=None, region=None, iam_role=None): + def _get_output_value( + self, stack_name, output_key, profile=None, region=None, sceptre_role=None + ): """ Attempts to get the Stack output named by ``output_key`` @@ -37,7 +36,7 @@ def _get_output_value(self, stack_name, output_key, profile=None, region=None, i :rtype: str :raises: sceptre.exceptions.DependencyStackMissingOutputError """ - outputs = self._get_stack_outputs(stack_name, profile, region, iam_role) + outputs = self._get_stack_outputs(stack_name, profile, region, sceptre_role) try: return outputs[output_key] @@ -48,7 +47,10 @@ def _get_output_value(self, stack_name, output_key, profile=None, region=None, i ) ) - def _get_stack_outputs(self, stack_name, profile=None, region=None, iam_role=None): + @functools.lru_cache(maxsize=4096) + def _get_stack_outputs( + self, stack_name, profile=None, region=None, sceptre_role=None + ): """ Communicates with AWS CloudFormation to fetch outputs from a specific Stack. @@ -59,9 +61,7 @@ def _get_stack_outputs(self, stack_name, profile=None, region=None, iam_role=Non :rtype: dict :raises: sceptre.stack.DependencyStackNotLaunchedException """ - self.logger.debug("Collecting outputs from '{0}'...".format( - stack_name - )) + self.logger.debug("Collecting outputs from '{0}'...".format(stack_name)) connection_manager = self.stack.connection_manager try: @@ -72,7 +72,7 @@ def _get_stack_outputs(self, stack_name, profile=None, region=None, iam_role=Non profile=profile, region=region, stack_name=stack_name, - iam_role=iam_role + sceptre_role=sceptre_role, ) except ClientError as e: if "does not exist" in e.response["Error"]["Message"]: @@ -85,8 +85,7 @@ def _get_stack_outputs(self, stack_name, profile=None, region=None, iam_role=Non self.logger.debug("Outputs: {0}".format(outputs)) formatted_outputs = dict( - (output["OutputKey"], output["OutputValue"]) - for output in outputs + (output["OutputKey"], output["OutputValue"]) for output in outputs ) return formatted_outputs @@ -125,13 +124,22 @@ def resolve(self): friendly_stack_name = self.dependency_stack_name.replace(TEMPLATE_EXTENSION, "") stack = next( - stack for stack in self.stack.dependencies if stack.name == friendly_stack_name + stack + for stack in self.stack.dependencies + if stack.name == friendly_stack_name ) - stack_name = "-".join([stack.project_code, friendly_stack_name.replace("/", "-")]) + stack_name = "-".join( + [stack.project_code, friendly_stack_name.replace("/", "-")] + ) - return self._get_output_value(stack_name, self.output_key, profile=stack.profile, - region=stack.region, iam_role=stack.iam_role) + return self._get_output_value( + stack_name, + self.output_key, + profile=stack.profile, + region=stack.region, + sceptre_role=stack.sceptre_role, + ) class StackOutputExternal(StackOutputBase): @@ -153,22 +161,23 @@ def resolve(self): :returns: The value of the Stack output. :rtype: str """ - self.logger.debug( - "Resolving external Stack output: {0}".format(self.argument) - ) + self.logger.debug("Resolving external Stack output: {0}".format(self.argument)) profile = None region = None - iam_role = None + sceptre_role = None arguments = shlex.split(self.argument) stack_argument = arguments[0] if len(arguments) > 1: extra_args = arguments[1].split("::", 2) - profile, region, iam_role = extra_args + (3 - len(extra_args)) * [None] + profile, region, sceptre_role = extra_args + (3 - len(extra_args)) * [None] dependency_stack_name, output_key = stack_argument.split("::") return self._get_output_value( - dependency_stack_name, output_key, - profile or None, region or None, iam_role or None + dependency_stack_name, + output_key, + profile or None, + region or None, + sceptre_role or None, ) diff --git a/sceptre/resolvers/sub.py b/sceptre/resolvers/sub.py new file mode 100644 index 000000000..93c29e905 --- /dev/null +++ b/sceptre/resolvers/sub.py @@ -0,0 +1,42 @@ +from sceptre.resolvers import Resolver + + +class Sub(Resolver): + """This resolver allows you to create a string using Python string format syntax. This is a + great way to combine together a number of resolver outputs into a single string. This functions + very similarly to Cloudformation's ``!Sub`` intrinsic function. + + The argument to this resolver should be a two-element list: (1) Is the format string, using + curly-brace templates to indicate variables, and (2) a dictionary where the keys are the format + string's variable names and the values are the variable values. + + Example: + + parameters: + ConnectionString: !sub + - "postgres://{username}:{password}@{hostname}:{port}/{database}" + - username: {{ var.username }} + password: !ssm /my/ssm/password + hostname: !stack_output my/database/stack.yaml::HostName + port: !stack_output my/database/stack.yaml::Port + database: {{var.database}} + """ + + def resolve(self): + error_message = ( + "The argument to !sub must be a two-element list, where the first element is the " + "a format string and the second element is a dict of values to interpolate into it." + ) + if not isinstance(self.argument, list) or len(self.argument) != 2: + self.raise_invalid_argument_error(error_message) + + template, variables = self.argument + if not isinstance(template, str) or not isinstance(variables, dict): + self.raise_invalid_argument_error(error_message) + + try: + return template.format(**variables) + except KeyError as e: + self.raise_invalid_argument_error( + f"Could not find !sub argument for {e}", e + ) diff --git a/sceptre/stack.py b/sceptre/stack.py index 395b5a9ec..b953dcb94 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -8,281 +8,425 @@ """ import logging -from typing import Mapping, Sequence +from typing import List, Any, Optional +import deprecation + +from sceptre import __version__ from sceptre.connection_manager import ConnectionManager -from sceptre.helpers import get_external_stack_name, sceptreise_path -from sceptre.hooks import HookProperty -from sceptre.resolvers import ResolvableProperty +from sceptre.exceptions import InvalidConfigFileError +from sceptre.helpers import ( + get_external_stack_name, + sceptreise_path, + create_deprecated_alias_property, +) +from sceptre.hooks import Hook, HookProperty +from sceptre.resolvers import ( + ResolvableContainerProperty, + ResolvableValueProperty, + RecursiveResolve, + PlaceholderType, +) from sceptre.template import Template -class Stack(object): +class Stack: """ Stack stores information about a particular CloudFormation Stack. :param name: The name of the Stack. - :type project: str :param project_code: A code which is prepended to the Stack names\ of all Stacks built by Sceptre. - :type project_code: str - :param template_path: The relative path to the CloudFormation, Jinja2\ - or Python template to build the Stack from. - :type template_path: str + :param template_path: The relative path to the CloudFormation, Jinja2, + or Python template to build the Stack from. If this is filled, + `template_handler_config` should not be filled. This field has been deprecated since + version 4.0.0 and will be removed in version 5.0.0. + + :param template_handler_config: Configuration for a Template Handler that can resolve + its arguments to a template string. Should contain the `type` property to specify + the type of template handler to load. Conflicts with `template_path`. :param region: The AWS region to build Stacks in. - :type region: str :param template_bucket_name: The name of the S3 bucket the Template is uploaded to. - :type template_bucket_name: str :param template_key_prefix: A prefix to the key used to store templates uploaded to S3 - :type template_key_prefix: str :param required_version: A PEP 440 compatible version specifier. If the Sceptre version does\ not fall within the given version requirement it will abort. - :type required_version: str :param parameters: The keys must match up with the name of the parameter.\ The value must be of the type as defined in the template. - :type parameters: dict :param sceptre_user_data: Data passed into\ `sceptre_handler(sceptre_user_data)` function in Python templates\ or accessible under `sceptre_user_data` variable within Jinja2\ templates. - :type sceptre_user_data: dict :param hooks: A list of arbitrary shell or python commands or scripts to\ run. - :type hooks: sceptre.hooks.Hook - :param s3_details: - :type s3_details: dict + :param s3_details: Details used for uploading templates to S3. :param dependencies: The relative path to the Stack, including the file\ extension of the Stack. - :type dependencies: list - :param role_arn: The ARN of a CloudFormation Service Role that is assumed\ + :param cloudformation_service_role: The ARN of a CloudFormation Service Role that is assumed\ by CloudFormation to create, update or delete resources. - :type role_arn: str :param protected: Stack protection against execution. - :type protected: bool :param tags: CloudFormation Tags to be applied to the Stack. - :type tags: dict - :param external_name: - :type external_name: str + :param external_name: The real stack name used for CloudFormation :param notifications: SNS topic ARNs to publish Stack related events to.\ A maximum of 5 ARNs can be specified per Stack. - :type notifications: list :param on_failure: This parameter describes the action taken by\ CloudFormation when a Stack fails to create. - :type on_failure: str - :param iam_role: The ARN of a role for Sceptre to assume before interacting\ + :param disable_rollback: If True, cloudformation will not rollback on deployment failures + + :param iam_role: The ARN of a role for Sceptre to assume before interacting + with the environment. If not supplied, Sceptre uses the user's AWS CLI + credentials. This field has been deprecated since version 4.0.0 and will be removed in + version 5.0.0. + + :param sceptre_role: The ARN of a role for Sceptre to assume before interacting\ with the environment. If not supplied, Sceptre uses the user's AWS CLI\ credentials. - :type iam_role: str + + :param iam_role_session_duration: The duration in seconds of the assumed IAM role session. + This field has been deprecated since version 4.0.0 and will be removed in version 5.0.0. + + :param sceptre_role_session_duration: The duration in seconds of the assumed IAM role session. :param profile: The name of the profile as defined in ~/.aws/config and\ ~/.aws/credentials. - :type profile: str :param stack_timeout: A timeout in minutes before considering the Stack\ deployment as failed. After the specified timeout, the Stack will\ - be rolled back. Specifiyng zero, as well as ommiting the field,\ + be rolled back. Specifying zero, as well as omitting the field,\ will result in no timeout. Supports only positive integer value. - :type stack_timeout: int - :param stack_group_config: The StackGroup config for the Stack - :type stack_group_config: dict + :param ignore: If True, this stack will be ignored during launches (but it can be explicitly + deployed with create, update, and delete commands. + + :param obsolete: If True, this stack will operate the same as if ignore was set, but it will + also be deleted if the prune command is invoked or the --prune option is used with the + launch command. + :param sceptre_role_session_duration: The session duration when Scetre assumes a role.\ + If not supplied, Sceptre uses default value (3600 seconds) + + :param stack_group_config: The StackGroup config for the Stack """ - parameters = ResolvableProperty("parameters") - _sceptre_user_data = ResolvableProperty("_sceptre_user_data") - notifications = ResolvableProperty("notifications") + parameters = ResolvableContainerProperty("parameters") + sceptre_user_data = ResolvableContainerProperty( + "sceptre_user_data", PlaceholderType.alphanum + ) + notifications = ResolvableContainerProperty("notifications") + tags = ResolvableContainerProperty("tags") + # placeholder_override=None here means that if the template_bucket_name is a resolver, + # placeholders have been enabled, and that stack hasn't been deployed yet, commands that would + # otherwise attempt to upload the template (like validate) won't actually use the template bucket + # and will act as if there was no template bucket set. + s3_details = ResolvableContainerProperty("s3_details", PlaceholderType.none) + template_handler_config = ResolvableContainerProperty( + "template_handler_config", PlaceholderType.alphanum + ) + + template_bucket_name = ResolvableValueProperty( + "template_bucket_name", PlaceholderType.none + ) + # Similarly, the placeholder_override=None for sceptre_role means that actions that would otherwise + # use the sceptre_role will act as if there was no iam role when the sceptre_role stack has not been + # deployed for commands that allow placeholders (like validate). + sceptre_role = ResolvableValueProperty("sceptre_role", PlaceholderType.none) + cloudformation_service_role = ResolvableValueProperty("cloudformation_service_role") + hooks = HookProperty("hooks") + iam_role = create_deprecated_alias_property( + "iam_role", "sceptre_role", "4.0.0", "5.0.0" + ) + role_arn = create_deprecated_alias_property( + "role_arn", "cloudformation_service_role", "4.0.0", "5.0.0" + ) + sceptre_role_session_duration = None + iam_role_session_duration = create_deprecated_alias_property( + "iam_role_session_duration", "sceptre_role_session_duration", "4.0.0", "5.0.0" + ) + def __init__( - self, name, project_code, template_path, region, template_bucket_name=None, - template_key_prefix=None, required_version=None, parameters=None, - sceptre_user_data=None, hooks=None, s3_details=None, iam_role=None, - dependencies=None, role_arn=None, protected=False, tags=None, - external_name=None, notifications=None, on_failure=None, profile=None, - stack_timeout=0, stack_group_config={} + self, + name: str, + project_code: str, + region: str, + template_path: str = None, + template_handler_config: dict = None, + template_bucket_name: str = None, + template_key_prefix: str = None, + required_version: str = None, + parameters: dict = None, + sceptre_user_data: dict = None, + hooks: Hook = None, + s3_details: dict = None, + sceptre_role: str = None, + iam_role: str = None, + dependencies: List["Stack"] = None, + cloudformation_service_role: str = None, + role_arn: str = None, + protected: bool = False, + tags: dict = None, + external_name: str = None, + notifications: List[str] = None, + on_failure: str = None, + disable_rollback=False, + profile: str = None, + stack_timeout: int = 0, + sceptre_role_session_duration: Optional[int] = None, + iam_role_session_duration: Optional[int] = None, + ignore=False, + obsolete=False, + stack_group_config: dict = {}, ): self.logger = logging.getLogger(__name__) self.name = sceptreise_path(name) self.project_code = project_code self.region = region - self.template_bucket_name = template_bucket_name - self.template_key_prefix = template_key_prefix self.required_version = required_version - self.external_name = external_name or get_external_stack_name(self.project_code, self.name) + self.external_name = external_name or get_external_stack_name( + self.project_code, self.name + ) + self.dependencies = dependencies or [] + self.protected = protected + self.on_failure = on_failure + self.disable_rollback = self._ensure_boolean( + "disable_rollback", disable_rollback + ) + self.stack_group_config = stack_group_config or {} + self.stack_timeout = stack_timeout + self.profile = profile + self.template_key_prefix = template_key_prefix + self._set_field_with_deprecated_alias( + "sceptre_role_session_duration", + sceptre_role_session_duration, + "iam_role_session_duration", + iam_role_session_duration, + ) + self.ignore = self._ensure_boolean("ignore", ignore) + self.obsolete = self._ensure_boolean("obsolete", obsolete) - self.template_path = template_path - self.s3_details = s3_details self._template = None self._connection_manager = None - self.protected = protected - self.role_arn = role_arn - self.on_failure = on_failure - self.dependencies = dependencies or [] + # Resolvers and hooks need to be assigned last + self.s3_details = s3_details + self._set_field_with_deprecated_alias( + "sceptre_role", sceptre_role, "iam_role", iam_role + ) self.tags = tags or {} - self.stack_timeout = stack_timeout - self.iam_role = iam_role - self.profile = profile - self.hooks = hooks or {} + self._set_field_with_deprecated_alias( + "cloudformation_service_role", + cloudformation_service_role, + "role_arn", + role_arn, + ) + self.template_bucket_name = template_bucket_name + self._set_field_with_deprecated_alias( + "template_handler_config", + template_handler_config, + "template_path", + template_path, + required=True, + preferred_config_name="template", + ) + + self.s3_details = s3_details self.parameters = parameters or {} - self._sceptre_user_data = sceptre_user_data or {} - self._sceptre_user_data_is_resolved = False + self.sceptre_user_data = sceptre_user_data or {} self.notifications = notifications or [] - self.stack_group_config = stack_group_config or {} + + self.hooks = hooks or {} + + def _ensure_boolean(self, config_name: str, value: Any) -> bool: + if not isinstance(value, bool): + raise InvalidConfigFileError( + f"{self.name}: Value for {config_name} must be a boolean, not a {type(value).__name__}" + ) + return value def __repr__(self): return ( "sceptre.stack.Stack(" - "name='{name}', " - "project_code={project_code}, " - "template_path={template_path}, " - "region={region}, " - "template_bucket_name={template_bucket_name}, " - "template_key_prefix={template_key_prefix}, " - "required_version={required_version}, " - "iam_role={iam_role}, " - "profile={profile}, " - "sceptre_user_data={sceptre_user_data}, " - "parameters={parameters}, " - "hooks={hooks}, " - "s3_details={s3_details}, " - "dependencies={dependencies}, " - "role_arn={role_arn}, " - "protected={protected}, " - "tags={tags}, " - "external_name={external_name}, " - "notifications={notifications}, " - "on_failure={on_failure}, " - "stack_timeout={stack_timeout}, " - "stack_group_config={stack_group_config}" - ")".format( - name=self.name, - project_code=self.project_code, - template_path=self.template_path, - region=self.region, - template_bucket_name=self.template_bucket_name, - template_key_prefix=self.template_key_prefix, - required_version=self.required_version, - iam_role=self.iam_role, - profile=self.profile, - sceptre_user_data=self.sceptre_user_data, - parameters=self.parameters, - hooks=self.hooks, - s3_details=self.s3_details, - dependencies=self.dependencies, - role_arn=self.role_arn, - protected=self.protected, - tags=self.tags, - external_name=self.external_name, - notifications=self.notifications, - on_failure=self.on_failure, - stack_timeout=self.stack_timeout, - stack_group_config=self.stack_group_config - ) + f"name='{self.name}', " + f"project_code={self.project_code}, " + f"template_handler_config={self.template_handler_config}, " + f"region={self.region}, " + f"template_bucket_name={self.template_bucket_name}, " + f"template_key_prefix={self.template_key_prefix}, " + f"required_version={self.required_version}, " + f"sceptre_role={self.sceptre_role}, " + f"sceptre_role_session_duration={self.sceptre_role_session_duration}, " + f"profile={self.profile}, " + f"sceptre_user_data={self.sceptre_user_data}, " + f"parameters={self.parameters}, " + f"hooks={self.hooks}, " + f"s3_details={self.s3_details}, " + f"dependencies={self.dependencies}, " + f"cloudformation_service_role={self.cloudformation_service_role}, " + f"protected={self.protected}, " + f"tags={self.tags}, " + f"external_name={self.external_name}, " + f"notifications={self.notifications}, " + f"on_failure={self.on_failure}, " + f"disable_rollback={self.disable_rollback}, " + f"stack_timeout={self.stack_timeout}, " + f"stack_group_config={self.stack_group_config}, " + f"ignore={self.ignore}, " + f"obsolete={self.obsolete}" + ")" ) def __str__(self): return self.name def __eq__(self, stack): + # We should not use any resolvable properties in __eq__, since it is used when adding the + # Stack to a set, which is done very early in plan resolution. Trying to reference resolvers + # before the plan is fully resolved can potentially blow up. return ( - self.name == stack.name and - self.project_code == stack.project_code and - self.template_path == stack.template_path and - self.region == stack.region and - self.template_bucket_name == stack.template_bucket_name and - self.template_key_prefix == stack.template_key_prefix and - self.required_version == stack.required_version and - self.iam_role == stack.iam_role and - self.profile == stack.profile and - self.sceptre_user_data == stack.sceptre_user_data and - self.parameters == stack.parameters and - self.hooks == stack.hooks and - self.s3_details == stack.s3_details and - self.dependencies == stack.dependencies and - self.role_arn == stack.role_arn and - self.protected == stack.protected and - self.tags == stack.tags and - self.external_name == stack.external_name and - self.notifications == stack.notifications and - self.on_failure == stack.on_failure and - self.stack_timeout == stack.stack_timeout and - self.stack_group_config == stack.stack_group_config + self.name == stack.name + and self.external_name == stack.external_name + and self.project_code == stack.project_code + and self.template_path == stack.template_path + and self.region == stack.region + and self.template_key_prefix == stack.template_key_prefix + and self.required_version == stack.required_version + and self.sceptre_role_session_duration + == stack.sceptre_role_session_duration + and self.profile == stack.profile + and self.dependencies == stack.dependencies + and self.protected == stack.protected + and self.on_failure == stack.on_failure + and self.disable_rollback == stack.disable_rollback + and self.stack_timeout == stack.stack_timeout + and self.ignore == stack.ignore + and self.obsolete == stack.obsolete ) def __hash__(self): return hash(str(self)) @property - def connection_manager(self): - """ - Returns ConnectionManager. - :returns: ConnectionManager. - :rtype: ConnectionManager + def connection_manager(self) -> ConnectionManager: + """Returns the ConnectionManager for the stack, creating it if it has not yet been created. + + :returns: ConnectionManager. """ if self._connection_manager is None: - self._connection_manager = ConnectionManager( - self.region, self.profile, self.external_name, self.iam_role + cache_connection_manager = True + try: + sceptre_role = self.sceptre_role + except RecursiveResolve: + # This would be the case when sceptre_role is set with a resolver (especially stack_output) + # that uses the stack's connection manager. This creates a temporary condition where + # you need the iam role to get the iam role. To get around this, it will temporarily + # use None as the sceptre_role but will re-attempt to resolve the value in future accesses. + # Since the Stack Output resolver (the most likely culprit) uses the target stack's + # sceptre_role rather than the current stack's one anyway, it actually doesn't matter, + # since the stack defining that sceptre_role won't actually be using that sceptre_role. + self.logger.debug( + "Resolving sceptre_role requires the Stack connection manager. Temporarily setting " + "the sceptre_role to None until it can be fully resolved." + ) + sceptre_role = None + cache_connection_manager = False + + connection_manager = ConnectionManager( + self.region, + self.profile, + self.external_name, + sceptre_role, + self.sceptre_role_session_duration, ) + if cache_connection_manager: + self._connection_manager = connection_manager + else: # Return early without caching the connection manager. + return connection_manager return self._connection_manager - @property - def sceptre_user_data(self): - """Returns sceptre_user_data after ensuring that it is fully resolved. - - :rtype: dict or list or None - """ - if not self._sceptre_user_data_is_resolved: - self._sceptre_user_data_is_resolved = True - self._resolve_sceptre_user_data() - return self._sceptre_user_data - @property def template(self): """ Returns the CloudFormation Template used to create the Stack. :returns: The Stack's template. - :rtype: str + :rtype: Template """ if self._template is None: self._template = Template( - path=self.template_path, + name=self.name, + handler_config=self.template_handler_config, sceptre_user_data=self.sceptre_user_data, + stack_group_config=self.stack_group_config, s3_details=self.s3_details, - connection_manager=self.connection_manager + connection_manager=self.connection_manager, ) return self._template - def _resolve_sceptre_user_data(self): - data = self._sceptre_user_data - if isinstance(data, Mapping): - iterator = data.values() - elif isinstance(data, Sequence): - iterator = data - else: - return - for value in iterator: - if isinstance(value, ResolvableProperty.ResolveLater): - value() + @property + @deprecation.deprecated( + "4.0.0", "5.0.0", __version__, "Use the template Stack Config key instead." + ) + def template_path(self) -> str: + """The path argument from the template_handler config. This field is deprecated as of v4.0.0 + and will be removed in v5.0.0. + """ + return self.template_handler_config["path"] + + @template_path.setter + @deprecation.deprecated( + "4.0.0", "5.0.0", __version__, "Use the template Stack Config key instead." + ) + def template_path(self, value: str): + self.template_handler_config = {"type": "file", "path": value} + + def _set_field_with_deprecated_alias( + self, + preferred_attribute_name, + preferred_value, + deprecated_attribute_name, + deprecated_value, + *, + required=False, + preferred_config_name=None, + deprecated_config_name=None, + ): + # This is a generic truthiness check. All current default values are falsy, so this should work. + # If we ever use this function where the default value is NOT falsy, this will be a problem. + preferred_config_name = preferred_config_name or preferred_attribute_name + deprecated_config_name = deprecated_config_name or deprecated_attribute_name + + if preferred_value and deprecated_value: + raise InvalidConfigFileError( + f"Both '{preferred_config_name}' and '{deprecated_config_name}' are set; You should only set a " + f"value for {preferred_config_name} because {deprecated_config_name} is deprecated." + ) + elif preferred_value: + setattr(self, preferred_attribute_name, preferred_value) + elif deprecated_value: + setattr(self, deprecated_attribute_name, deprecated_value) + elif required: + raise InvalidConfigFileError( + f"{preferred_config_name} is a required Stack Config." + ) + else: # In case they're both falsy, we should just set the value using the preferred value. + setattr(self, preferred_attribute_name, preferred_value) diff --git a/sceptre/stack_status.py b/sceptre/stack_status.py index 67bddb1da..34cc516ac 100644 --- a/sceptre/stack_status.py +++ b/sceptre/stack_status.py @@ -12,6 +12,7 @@ class StackStatus(object): """ StackStatus stores simplified Stack statuses. """ + COMPLETE = "complete" FAILED = "failed" IN_PROGRESS = "in progress" @@ -22,6 +23,7 @@ class StackChangeSetStatus(object): """ StackChangeSetStatus stores simplified ChangeSet statuses. """ + PENDING = "pending" READY = "ready" DEFUNCT = "defunct" diff --git a/sceptre/stack_status_colourer.py b/sceptre/stack_status_colourer.py index a0b9a2725..965bac3c8 100644 --- a/sceptre/stack_status_colourer.py +++ b/sceptre/stack_status_colourer.py @@ -14,6 +14,8 @@ class StackStatusColourer(object): """ StackStatusColourer adds colours to stack statuses. + These are documented here: + https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html """ STACK_STATUS_CODES = { @@ -24,6 +26,12 @@ class StackStatusColourer(object): "DELETE_COMPLETE": Fore.GREEN, "DELETE_FAILED": Fore.RED, "DELETE_IN_PROGRESS": Fore.YELLOW, + "IMPORT_COMPLETE": Fore.GREEN, + "IMPORT_IN_PROGRESS": Fore.YELLOW, + "IMPORT_ROLLBACK_COMPLETE": Fore.GREEN, + "IMPORT_ROLLBACK_FAILED": Fore.RED, + "IMPORT_ROLLBACK_IN_PROGRESS": Fore.YELLOW, + "REVIEW_IN_PROGRESS": Fore.YELLOW, "ROLLBACK_COMPLETE": Fore.RED, "ROLLBACK_FAILED": Fore.RED, "ROLLBACK_IN_PROGRESS": Fore.YELLOW, @@ -34,12 +42,10 @@ class StackStatusColourer(object): "UPDATE_ROLLBACK_COMPLETE": Fore.GREEN, "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS": Fore.YELLOW, "UPDATE_ROLLBACK_FAILED": Fore.RED, - "UPDATE_ROLLBACK_IN_PROGRESS": Fore.YELLOW + "UPDATE_ROLLBACK_IN_PROGRESS": Fore.YELLOW, } - STACK_STATUS_PATTERN = re.compile( - r"\b({0})\b".format("|".join(STACK_STATUS_CODES)) - ) + STACK_STATUS_PATTERN = re.compile(r"\b({0})\b".format("|".join(STACK_STATUS_CODES))) def colour(self, string): """ @@ -60,6 +66,6 @@ def colour(self, string): "{0}{1}{2}".format( self.STACK_STATUS_CODES[status], status, Style.RESET_ALL ), - string + string, ) return string diff --git a/sceptre/template.py b/sceptre/template.py index 8084cf16f..dcb966fb0 100644 --- a/sceptre/template.py +++ b/sceptre/template.py @@ -7,20 +7,15 @@ and implements methods for uploading it to S3. """ -import imp import logging -import os -import sys import threading -import traceback - import botocore -from jinja2 import Environment -from jinja2 import FileSystemLoader -from jinja2 import StrictUndefined -from jinja2 import select_autoescape -from sceptre.exceptions import UnsupportedTemplateFileTypeError -from sceptre.exceptions import TemplateSceptreHandlerError +import sys + +import sceptre.helpers + +from sceptre.exceptions import TemplateHandlerNotFoundError +from sceptre.logging import StackLoggerAdapter class Template(object): @@ -29,14 +24,19 @@ class Template(object): loading, storing and optionally uploading local templates for use by CloudFormation. - :param path: The absolute path to the file which stores the CloudFormation\ - template. - :type path: str + :param name: The name of the template. Should be safe to use in filenames and not contain path segments. + :type name: str + + :param handler_config: The configuration for a Template handler. Must contain a `type`. + :type handler_config: dict :param sceptre_user_data: A dictionary of arbitrary data to be passed to\ a handler function in an external Python script. :type sceptre_user_data: dict + :param stack_group_config: The StackGroup config for the Stack. + :type stack_group_config: dict + :param connection_manager: :type connection_manager: sceptre.connection_manager.ConnectionManager @@ -47,61 +47,34 @@ class Template(object): _boto_s3_lock = threading.Lock() def __init__( - self, path, sceptre_user_data, connection_manager=None, s3_details=None + self, + name, + handler_config, + sceptre_user_data, + stack_group_config, + connection_manager=None, + s3_details=None, ): - self.logger = logging.getLogger(__name__) - - self.path = path + self.logger = StackLoggerAdapter(logging.getLogger(__name__), name) + self.name = name + self.handler_config = handler_config + if self.handler_config is not None and self.handler_config.get("type") is None: + self.handler_config["type"] = "file" self.sceptre_user_data = sceptre_user_data + self.stack_group_config = stack_group_config self.connection_manager = connection_manager self.s3_details = s3_details - self.name = os.path.basename(path).split(".")[0] + self._registry = None self._body = None def __repr__(self): - return ( - "sceptre.template.Template(name='{0}', path='{1}', " - "sceptre_user_data={2}, s3_details={3})".format( - self.name, self.path, self.sceptre_user_data, self.s3_details - ) + return sceptre.helpers.gen_repr( + self, + class_label="sceptre.template.Template", + attributes=["name", "handler_config", "sceptre_user_data", "s3_details"], ) - def _print_template_traceback(self): - """ - Prints a stack trace, including only files which are inside a - 'templates' directory. The function is intended to give the operator - instant feedback about why their templates are failing to compile. - - :rtype: None - """ - def _print_frame(filename, line, fcn, line_text): - self.logger.error("{}:{}: Template error in '{}'\n=> `{}`".format( - filename, line, fcn, line_text)) - - try: - _, _, tb = sys.exc_info() - stack_trace = traceback.extract_tb(tb) - search_string = os.path.join('', 'templates', '') - if search_string in self.path: - template_path = self.path.split(search_string)[0] + search_string - else: - return - for frame in stack_trace: - if isinstance(frame, tuple): - # Python 2 / Old style stack frame - if template_path in frame[0]: - _print_frame(frame[0], frame[1], frame[2], frame[3]) - else: - if template_path in frame.filename: - _print_frame(frame.filename, frame.lineno, frame.name, frame.line) - except Exception as tb_exception: - self.logger.error( - 'A template error occured. ' + - 'Additionally, a traceback exception occured. Exception: %s', - tb_exception - ) - @property def body(self): """ @@ -111,78 +84,25 @@ def body(self): :rtype: str """ if self._body is None: - file_extension = os.path.splitext(self.path)[1] - - try: - if file_extension in {".json", ".yaml", ".template"}: - with open(self.path) as template_file: - self._body = template_file.read() - elif file_extension == ".j2": - self._body = self._render_jinja_template( - os.path.dirname(self.path), - os.path.basename(self.path), - {"sceptre_user_data": self.sceptre_user_data} - ) - elif file_extension == ".py": - self._body = self._call_sceptre_handler() - - else: - raise UnsupportedTemplateFileTypeError( - "Template has file extension %s. Only .py, .yaml, " - ".template, .json and .j2 are supported.", - os.path.splitext(self.path)[1] - ) - except Exception as e: - self._print_template_traceback() - raise e + type = self.handler_config.get("type") + handler_class = self._get_handler_of_type(type) + handler = handler_class( + name=self.name, + arguments={k: v for k, v in self.handler_config.items() if k != "type"}, + sceptre_user_data=self.sceptre_user_data, + connection_manager=self.connection_manager, + stack_group_config=self.stack_group_config, + ) + handler.validate() + body = handler.handle() + if isinstance(body, bytes): + body = body.decode("utf-8") + if not str(body).startswith("---"): + body = "---\n{}".format(body) + self._body = body return self._body - def _call_sceptre_handler(self): - """ - Calls the function `sceptre_handler` within templates that are python - scripts. - - :returns: The string returned from sceptre_handler in the template. - :rtype: str - :raises: IOError - :raises: TemplateSceptreHandlerError - """ - # Get relative path as list between current working directory and where - # the template is - # NB: this is a horrible hack... - relpath = os.path.relpath(self.path, os.getcwd()).split(os.path.sep) - relpaths_to_add = [ - os.path.sep.join(relpath[:i+1]) - for i in range(len(relpath[:-1])) - ] - # Add any directory between the current working directory and where - # the template is to the python path - for directory in relpaths_to_add: - sys.path.append(os.path.join(os.getcwd(), directory)) - self.logger.debug( - "%s - Getting CloudFormation from %s", self.name, self.path - ) - - if not os.path.isfile(self.path): - raise IOError("No such file or directory: '%s'", self.path) - - module = imp.load_source(self.name, self.path) - - try: - body = module.sceptre_handler(self.sceptre_user_data) - except AttributeError as e: - if 'sceptre_handler' in str(e): - raise TemplateSceptreHandlerError( - "The template does not have the required " - "'sceptre_handler(sceptre_user_data)' function." - ) - else: - raise e - for directory in relpaths_to_add: - sys.path.remove(os.path.join(os.getcwd(), directory)) - return body - def upload_to_s3(self): """ Uploads the template to ``bucket_name`` and returns its URL. @@ -203,11 +123,13 @@ def upload_to_s3(self): # Remove any leading or trailing slashes the user may have added. bucket_name = self.s3_details["bucket_name"] bucket_key = self.s3_details["bucket_key"] - bucket_region = self.s3_details["bucket_region"] + bucket_region = self._bucket_region(bucket_name) self.logger.debug( "%s - Uploading template to: 's3://%s/%s'", - self.name, bucket_name, bucket_key + self.name, + bucket_name, + bucket_key, ) self.connection_manager.call( service="s3", @@ -216,20 +138,16 @@ def upload_to_s3(self): "Bucket": bucket_name, "Key": bucket_key, "Body": self.body, - "ServerSideEncryption": "AES256" - } + "ServerSideEncryption": "AES256", + }, ) - china_regions = ["cn-north-1", "cn-northwest-1"] - - if bucket_region in china_regions: - url = "https://{0}.s3.{1}.amazonaws.com.cn/{2}".format( - bucket_name, bucket_region, bucket_key - ) - else: - url = "https://{0}.s3.amazonaws.com/{1}".format( - bucket_name, bucket_key - ) + url = "https://{}.s3.{}.amazonaws.{}/{}".format( + bucket_name, + bucket_region, + self._domain_from_region(bucket_region), + bucket_key, + ) self.logger.debug("%s - Template URL: '%s'", self.name, url) @@ -246,26 +164,19 @@ def _bucket_exists(self): """ bucket_name = self.s3_details["bucket_name"] self.logger.debug( - "%s - Attempting to find template bucket '%s'", - self.name, bucket_name + "%s - Attempting to find template bucket '%s'", self.name, bucket_name ) try: self.connection_manager.call( - service="s3", - command="head_bucket", - kwargs={"Bucket": bucket_name} + service="s3", command="head_bucket", kwargs={"Bucket": bucket_name} ) except botocore.exceptions.ClientError as exp: if exp.response["Error"]["Message"] == "Not Found": - self.logger.debug( - "%s - %s bucket not found.", self.name, bucket_name - ) + self.logger.debug("%s - %s bucket not found.", self.name, bucket_name) return False else: raise - self.logger.debug( - "%s - Found template bucket '%s'", self.name, bucket_name - ) + self.logger.debug("%s - Found template bucket '%s'", self.name, bucket_name) return True def _create_bucket(self): @@ -277,15 +188,11 @@ def _create_bucket(self): """ bucket_name = self.s3_details["bucket_name"] - self.logger.debug( - "%s - Creating new bucket '%s'", self.name, bucket_name - ) + self.logger.debug("%s - Creating new bucket '%s'", self.name, bucket_name) if self.connection_manager.region == "us-east-1": self.connection_manager.call( - service="s3", - command="create_bucket", - kwargs={"Bucket": bucket_name} + service="s3", command="create_bucket", kwargs={"Bucket": bucket_name} ) else: self.connection_manager.call( @@ -295,8 +202,8 @@ def _create_bucket(self): "Bucket": bucket_name, "CreateBucketConfiguration": { "LocationConstraint": self.connection_manager.region - } - } + }, + }, ) def get_boto_call_parameter(self): @@ -309,39 +216,58 @@ def get_boto_call_parameter(self): :returns: The boto call parameter for the template. :rtype: dict """ - if self.s3_details: + # If bucket_name is set to None, it should be ignored and not uploaded. + if self.s3_details and self.s3_details.get("bucket_name"): url = self.upload_to_s3() return {"TemplateURL": url} else: return {"TemplateBody": self.body} + def _bucket_region(self, bucket_name): + region = self.connection_manager.call( + service="s3", command="get_bucket_location", kwargs={"Bucket": bucket_name} + ).get("LocationConstraint") + return region if region else "us-east-1" + + @staticmethod + def _domain_from_region(region): + return "com.cn" if region.startswith("cn-") else "com" + @staticmethod - def _render_jinja_template(template_dir, filename, jinja_vars): + def _iterate_entry_points(group, name): + """ + Helper to determine whether to use pkg_resources or importlib.metadata. + https://docs.python.org/3/library/importlib.metadata.html """ - Renders a jinja template. + if sys.version_info < (3, 10): + from pkg_resources import iter_entry_points - Sceptre supports passing sceptre_user_data to JSON and YAML - CloudFormation templates using Jinja2 templating. + return iter_entry_points(group, name) + else: + from importlib.metadata import entry_points - :param template_dir: The directory containing the template. - :type template_dir: str - :param filename: The name of the template file. - :type filename: str - :param jinja_vars: Dict of variables to render into the template. - :type jinja_vars: dict - :returns: The body of the CloudFormation template. - :rtype: str + return entry_points(group=group, name=name) + + def _get_handler_of_type(self, type): """ - logger = logging.getLogger(__name__) - logger.debug("%s Rendering CloudFormation template", filename) - env = Environment( - autoescape=select_autoescape( - disabled_extensions=('j2',), - default=True, - ), - loader=FileSystemLoader(template_dir), - undefined=StrictUndefined - ) - template = env.get_template(filename) - body = template.render(**jinja_vars) - return body + Gets a TemplateHandler type from the registry that can be used to get a string + representation of a CloudFormation template. + :param type: The type of Template Handler to load + :type type: str + :return: Instantiated TemplateHandler + :rtype: class + """ + if not self._registry: + self._registry = {} + + for entry_point in self._iterate_entry_points( + "sceptre.template_handlers", type + ): + self._registry[entry_point.name] = entry_point.load() + + if type not in self._registry: + raise TemplateHandlerNotFoundError( + 'Handler of type "{0}" not found'.format(type) + ) + + return self._registry[type] diff --git a/sceptre/template_handlers/__init__.py b/sceptre/template_handlers/__init__.py new file mode 100644 index 000000000..12873a0e9 --- /dev/null +++ b/sceptre/template_handlers/__init__.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +import abc +import logging + +import six +from jsonschema import validate, ValidationError + +from sceptre.exceptions import TemplateHandlerArgumentsInvalidError +from sceptre.logging import StackLoggerAdapter + + +@six.add_metaclass(abc.ABCMeta) +class TemplateHandler: + """ + TemplateHandler is an abstract base class that should be inherited + by all Template Handlers. + + :param name: Name of the template + :type name: str + + :param arguments: The arguments of the template handler + :type arguments: dict + + :param sceptre_user_data: Sceptre user data in stack config + :type sceptre_user_data: dict + + :param connection_manager: Connection manager used to call AWS + :type connection_manager: sceptre.connection_manager.ConnectionManager + + :param stack_group_config: The Stack group config to use as defaults. + :type stack_group_config: dict + """ + + __metaclass__ = abc.ABCMeta + + standard_template_extensions = [".json", ".yaml", ".template"] + jinja_template_extensions = [".j2"] + python_template_extensions = [".py"] + supported_template_extensions = ( + standard_template_extensions + + jinja_template_extensions + + python_template_extensions + ) + + def __init__( + self, + name, + arguments=None, + sceptre_user_data=None, + connection_manager=None, + stack_group_config=None, + ): + self.logger = StackLoggerAdapter(logging.getLogger(__name__), name) + self.name = name + self.arguments = arguments + self.sceptre_user_data = sceptre_user_data + self.connection_manager = connection_manager + + if stack_group_config is None: + stack_group_config = {} + self.stack_group_config = stack_group_config + + @abc.abstractmethod + def schema(self): + """ + Returns the schema for the arguments of this Template Resolver. This will + be used to validate that the arguments passed in the stack config are what + the Template Handler expects. + :return: JSON schema that can be validated + :rtype: object + """ + pass + + @abc.abstractmethod + def handle(self): + """ + An abstract method which must be overwritten by all inheriting classes. + This method is called to retrieve the template. + Implementation of this method in subclasses must return a string that + can be interpreted by Sceptre (CloudFormation YAML / JSON, Jinja or Python) + """ + pass # pragma: no cover + + def validate(self): + """ + Validates if the current arguments are correct according to the schema. If this + does not raise an exception, the template handler's arguments are valid. + """ + try: + validate(instance=self.arguments, schema=self.schema()) + except ValidationError as e: + raise TemplateHandlerArgumentsInvalidError(e) diff --git a/sceptre/template_handlers/file.py b/sceptre/template_handlers/file.py new file mode 100644 index 000000000..9db6605bd --- /dev/null +++ b/sceptre/template_handlers/file.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import sceptre.template_handlers.helper as helper + +from os import path +from pathlib import Path + +from sceptre.exceptions import UnsupportedTemplateFileTypeError +from sceptre.template_handlers import TemplateHandler +from sceptre.helpers import normalise_path + + +class File(TemplateHandler): + """ + Template handler that can load files from disk. Supports JSON, YAML, Jinja2 and Python. + """ + + def __init__(self, *args, **kwargs): + super(File, self).__init__(*args, **kwargs) + + def schema(self): + return { + "type": "object", + "properties": { + "path": {"type": "string"}, + }, + "required": ["path"], + } + + def handle(self): + input_path = Path(self.arguments["path"]) + path = self._resolve_template_path(str(input_path)) + + if input_path.suffix not in self.supported_template_extensions: + raise UnsupportedTemplateFileTypeError( + "Template has file extension %s. Only %s are supported.", + input_path.suffix, + ",".join(self.supported_template_extensions), + ) + + try: + if input_path.suffix in self.standard_template_extensions: + with open(path) as template_file: + return template_file.read() + elif input_path.suffix in self.jinja_template_extensions: + return helper.render_jinja_template( + path, + {"sceptre_user_data": self.sceptre_user_data}, + self.stack_group_config.get("j2_environment", {}), + ) + elif input_path.suffix in self.python_template_extensions: + return helper.call_sceptre_handler(path, self.sceptre_user_data) + except Exception as e: + helper.print_template_traceback(path) + raise e + + def _resolve_template_path(self, template_path): + """ + Return the project_path joined to template_path as + a string. + + Note that os.path.join defers to an absolute path + if the input is absolute. + """ + return path.join( + self.stack_group_config["project_path"], + "templates", + normalise_path(template_path), + ) diff --git a/sceptre/template_handlers/helper.py b/sceptre/template_handlers/helper.py new file mode 100644 index 000000000..c02fb7c31 --- /dev/null +++ b/sceptre/template_handlers/helper.py @@ -0,0 +1,146 @@ +import logging +import os +import sys +import traceback + +from importlib.machinery import SourceFileLoader +from jinja2 import Environment, select_autoescape, FileSystemLoader, StrictUndefined +from pathlib import Path +from sceptre.exceptions import TemplateSceptreHandlerError, TemplateNotFoundError +from sceptre.config import strategies + +logger = logging.getLogger(__name__) + +""" +Template handler helpers. +""" + + +def call_sceptre_handler(path, sceptre_user_data): + """ + Calls the function `sceptre_handler` within templates that are python + scripts. + + :param path: A path to the file. + :type name: str + :param sceptre_user_data: The sceptre_user_data parameter values. + :type name: str + :returns: The string returned from sceptre_handler in the template. + :rtype: str + :raises: IOError + :raises: TemplateSceptreHandlerError + """ + # Get relative path as list between current working directory and where + # the template is + # NB: this is a horrible hack... + relpath = os.path.relpath(path, os.getcwd()).split(os.path.sep) + relpaths_to_add = [ + os.path.sep.join(relpath[: i + 1]) for i in range(len(relpath[:-1])) + ] + # Add any directory between the current working directory and where + # the template is to the python path + for directory in relpaths_to_add: + sys.path.append(os.path.join(os.getcwd(), directory)) + logger.debug("Getting CloudFormation from %s", path) + + if not os.path.isfile(path): + raise TemplateNotFoundError("No such template file: '%s'", path) + + module = SourceFileLoader(path, path).load_module() + + try: + body = module.sceptre_handler(sceptre_user_data) + except AttributeError as e: + if "sceptre_handler" in str(e): + raise TemplateSceptreHandlerError( + "The template does not have the required " + "'sceptre_handler(sceptre_user_data)' function." + ) + else: + raise e + for directory in relpaths_to_add: + sys.path.remove(os.path.join(os.getcwd(), directory)) + return body + + +def print_template_traceback(path): + """ + Prints a stack trace, including only files which are inside a + 'templates' directory. The function is intended to give the operator + instant feedback about why their templates are failing to compile. + + :param path: A path to the file. + :type name: str + :rtype: None + """ + + def _print_frame(filename, line, fcn, line_text): + logger.error( + "{}:{}: Template error in '{}'\n=> `{}`".format( + filename, line, fcn, line_text + ) + ) + + try: + _, _, tb = sys.exc_info() + stack_trace = traceback.extract_tb(tb) + search_string = os.path.join("", "templates", "") + if search_string in path: + template_path = path.split(search_string)[0] + search_string + else: + return + for frame in stack_trace: + if isinstance(frame, tuple): + # Python 2 / Old style stack frame + if template_path in frame[0]: + _print_frame(frame[0], frame[1], frame[2], frame[3]) + else: + if template_path in frame.filename: + _print_frame(frame.filename, frame.lineno, frame.name, frame.line) + except Exception as tb_exception: + logger.error( + "A template error occured. " + + "Additionally, a traceback exception occured. Exception: %s", + tb_exception, + ) + + +def render_jinja_template(path, jinja_vars, j2_environment): + """ + Renders a jinja template. + + Sceptre supports passing sceptre_user_data to JSON and YAML + CloudFormation templates using Jinja2 templating. + + :param path: The path to the template file. + :type path: str + :param jinja_vars: Dict of variables to render into the template. + :type jinja_vars: dict + :param j2_environment: The jinja2 environment. + :type stack_group_config: dict + + :returns: The body of the CloudFormation template. + :rtype: str + """ + path = Path(path) + if not path.exists(): + raise TemplateNotFoundError("No such template file: '%s'", path) + + logger.debug("%s Rendering CloudFormation template", path) + default_j2_environment_config = { + "autoescape": select_autoescape( + disabled_extensions=("j2",), + default=True, + ), + "loader": FileSystemLoader(path.parent), + "undefined": StrictUndefined, + } + j2_environment_config = strategies.dict_merge( + default_j2_environment_config, j2_environment + ) + j2_environment = Environment(**j2_environment_config) + + template = j2_environment.get_template(path.name) + + body = template.render(**jinja_vars) + return body diff --git a/sceptre/template_handlers/http.py b/sceptre/template_handlers/http.py new file mode 100644 index 000000000..e1387c72a --- /dev/null +++ b/sceptre/template_handlers/http.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +import pathlib +import tempfile +from urllib.parse import urlparse + +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +import sceptre.template_handlers.helper as helper +from sceptre.exceptions import UnsupportedTemplateFileTypeError +from sceptre.template_handlers import TemplateHandler + +HANDLER_OPTION_KEY = "http_template_handler" +HANDLER_RETRIES_OPTION_PARAM = "retries" +DEFAULT_RETRIES_OPTION = 5 +HANDLER_TIMEOUT_OPTION_PARAM = "timeout" +DEFAULT_TIMEOUT_OPTION = 5 + + +class Http(TemplateHandler): + """ + Template handler that can resolve templates from the web. Standard CFN templates + with extension (.json, .yaml, .template) are deployed directly from memory + while references to jinja (.j2) and python (.py) templates are downloaded, + transformed into CFN templates then deployed to AWS. + """ + + def __init__(self, *args, **kwargs): + super(Http, self).__init__(*args, **kwargs) + + def schema(self): + return { + "type": "object", + "properties": {"url": {"type": "string"}}, + "required": ["url"], + } + + def handle(self): + """ + handle template from web + """ + url = self.arguments["url"] + path = pathlib.Path(urlparse(url).path) + + if path.suffix not in self.supported_template_extensions: + raise UnsupportedTemplateFileTypeError( + "Template has file extension %s. Only %s are supported.", + path.suffix, + ",".join(self.supported_template_extensions), + ) + + retries = self._get_handler_option( + HANDLER_RETRIES_OPTION_PARAM, DEFAULT_RETRIES_OPTION + ) + timeout = self._get_handler_option( + HANDLER_TIMEOUT_OPTION_PARAM, DEFAULT_TIMEOUT_OPTION + ) + try: + template = self._get_template(url, retries=retries, timeout=timeout) + if ( + path.suffix + in self.jinja_template_extensions + self.python_template_extensions + ): + file = tempfile.NamedTemporaryFile(prefix=path.stem) + self.logger.debug("Template file saved to: %s", file.name) + with file as f: + f.write(template) + f.seek(0) + f.read() + if path.suffix in self.jinja_template_extensions: + template = helper.render_jinja_template( + f.name, + {"sceptre_user_data": self.sceptre_user_data}, + self.stack_group_config.get("j2_environment", {}), + ) + elif path.suffix in self.python_template_extensions: + template = helper.call_sceptre_handler( + f.name, self.sceptre_user_data + ) + + except Exception as e: + helper.print_template_traceback(path) + raise e + + return template + + def _get_template(self, url: str, retries: int, timeout: int) -> str: + """ + Get template from the web + :param url: The url to the template + :param retries: The number of retry attempts. + :param timeout: The timeout for the session in seconds. + :raises: :class:`requests.exceptions.HTTPError`: When a download error occurs + """ + self.logger.debug("Downloading file from: %s", url) + session = self._get_retry_session(retries=retries) + response = session.get(url, timeout=timeout) + + # If the response was unsuccessful, raise an error. + response.raise_for_status() + + return response.content + + def _get_retry_session( + self, + retries, + backoff_factor=0.3, + status_forcelist=(429, 500, 502, 503, 504), + session=None, + ): + """ + Get a request session with retries. Retry options are explained in the request libraries + https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry + """ + session = session or requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + def _get_handler_option(self, name, default): + """ + Get the template handler options + :param url: The option name + :type: str + :param default: The default value if option is not set. + :rtype: int + """ + if HANDLER_OPTION_KEY in self.stack_group_config: + option = self.stack_group_config.get(HANDLER_OPTION_KEY) + if name in option: + return option.get(name) + + return default diff --git a/sceptre/template_handlers/s3.py b/sceptre/template_handlers/s3.py new file mode 100644 index 000000000..0725a2315 --- /dev/null +++ b/sceptre/template_handlers/s3.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +import pathlib +import tempfile +import sceptre.template_handlers.helper as helper + +from sceptre.exceptions import UnsupportedTemplateFileTypeError +from sceptre.template_handlers import TemplateHandler + + +class S3(TemplateHandler): + """ + Template handler that can resolve templates from S3. Raw CFN templates + with extension (.json, .yaml, .template) are deployed directly from memory + while references to jinja (.j2) and python (.py) templates are downloaded, + transformed into CFN templates then deployed to AWS. + """ + + def __init__(self, *args, **kwargs): + super(S3, self).__init__(*args, **kwargs) + + def schema(self): + return { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + } + + def handle(self): + """ + handle template in S3 bucket + """ + input_path = self.arguments["path"] + path = pathlib.Path(input_path) + + standard_template_suffix = [".json", ".yaml", ".template"] + jinja_template_suffix = [".j2"] + python_template_suffix = [".py"] + supported_suffix = ( + standard_template_suffix + jinja_template_suffix + python_template_suffix + ) + + if path.suffix not in supported_suffix: + raise UnsupportedTemplateFileTypeError( + "Template has file extension %s. Only %s are supported.", + path.suffix, + ",".join(supported_suffix), + ) + + try: + template = self._get_template(path) + if path.suffix in jinja_template_suffix + python_template_suffix: + file = tempfile.NamedTemporaryFile(prefix=path.stem) + with file as f: + f.write(template) + f.seek(0) + f.read() + if path.suffix in jinja_template_suffix: + template = helper.render_jinja_template( + f.name, + {"sceptre_user_data": self.sceptre_user_data}, + self.stack_group_config.get("j2_environment", {}), + ) + elif path.suffix in python_template_suffix: + template = helper.call_sceptre_handler( + f.name, self.sceptre_user_data + ) + + except Exception as e: + helper.print_template_traceback(path) + raise e + + return template + + def _get_template(self, path): + """ + Get template from S3 bucket + + :param path: The path to the object in the bucket + :type: str + :returns: The body of the CloudFormation template. + :rtype: str + """ + self.logger.debug("Downloading file from S3: %s", path) + bucket = path.parts[0] + key = "/".join(path.parts[1:]) + + try: + response = self.connection_manager.call( + service="s3", + command="get_object", + kwargs={"Bucket": bucket, "Key": key}, + ) + return response["Body"].read() + except Exception as e: + self.logger.critical(e) + raise e diff --git a/setup.cfg b/setup.cfg index b6f761ad9..f73632845 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,16 +1,16 @@ [bumpversion] -current_version = 2.2.1 +current_version = 4.0.2 parse = (?P\d+)\.(?P\d+)\.(?P\d+)|(?P.*) commit = True tag = True -serialize = +serialize = {major}.{minor}.{patch}{release_candidate} {major}.{minor}.{patch} [bumpversion:file:sceptre/__init__.py] [bumpversion:part:release_candidate] -values = +values = rc0 rc1 rc2 @@ -22,18 +22,3 @@ universal = 1 [aliases] test = pytest - -[flake8] -exclude = - .git, - __pycache__, - build, - dist, - .tox, - venv -max-complexity = 12 -per-file-ignores = - docs/_api/conf.py: E265 - integration-tests/steps/*: E501,F811,F403,F405 -max-line-length = 99 - diff --git a/setup.py b/setup.py index 57fc7a5d8..3a96a2405 100755 --- a/setup.py +++ b/setup.py @@ -1,64 +1,67 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from sceptre import __version__ +import codecs +import os from setuptools import setup, find_packages -from os import path -with open("README.md") as readme_file: - readme = readme_file.read() -with open("CHANGELOG.md") as history_file: - history = history_file.read() +def read_file(rel_path): + here = os.path.abspath(os.path.dirname(__file__)) + with codecs.open(os.path.join(here, rel_path), "r") as fp: + return fp.read() + + +def get_version(rel_path): + for line in read_file(rel_path).splitlines(): + if line.startswith("__version__"): + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + else: + raise RuntimeError("Unable to find version string.") + install_requirements = [ "boto3>=1.3,<2.0", - "click==7.0", + "click>=7.0,<9.0", + "cfn-flip>=1.2.3,<2.0", + "deepdiff>=5.5.0,<6.0", + "deprecation>=2.0.0,<3.0", "PyYaml>=5.1,<6.0", - "Jinja2>=2.8,<3", - "colorama==0.3.9", - "packaging==16.8", + "Jinja2>=3.0,<4", + "jsonschema>=3.2,<3.3", + "colorama>=0.3.9", + "packaging>=16.8,<22.0", + "sceptre-cmd-resolver>=1.1.3,<3", + "sceptre-file-resolver>=1.0.4,<2", "six>=1.11.0,<2.0.0", - "networkx==2.1", - "typing>=3.7.0,<3.8.0" + "networkx>=2.6,<2.7", ] -test_requirements = [ - "pytest>=3.2", - "troposphere>=2.0.0", - "moto==1.3.8", - "mock==2.0.0", - "behave==1.2.5", - "freezegun==0.3.12" -] +extra_requirements = { + "troposphere": ["troposphere>=4,<5"], +} -setup_requirements = [ - "pytest-runner>=3" -] setup( name="sceptre", - version=__version__, + version=get_version("sceptre/__init__.py"), description="Cloud Provisioning Tool", - long_description=readme, + long_description=read_file("README.md"), long_description_content_type="text/markdown", author="Cloudreach", author_email="sceptre@cloudreach.com", - license='Apache2', - url="https://github.com/cloudreach/sceptre", + license="Apache2", + url="https://github.com/Sceptre/sceptre", packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), - package_dir={ - "sceptre": "sceptre" - }, + package_dir={"sceptre": "sceptre"}, py_modules=["sceptre"], entry_points={ - "console_scripts": [ - 'sceptre = sceptre.cli:cli' - ], + "console_scripts": ["sceptre = sceptre.cli:cli"], "sceptre.hooks": [ "asg_scheduled_actions =" "sceptre.hooks.asg_scaling_processes:ASGScalingProcesses", - "cmd = sceptre.hooks.cmd:Cmd" + "cmd = sceptre.hooks.cmd:Cmd", ], "sceptre.resolvers": [ "environment_variable =" @@ -66,14 +69,28 @@ "file_contents = sceptre.resolvers.file_contents:FileContents", "stack_output = sceptre.resolvers.stack_output:StackOutput", "stack_output_external =" - "sceptre.resolvers.stack_output:StackOutputExternal" - ] + "sceptre.resolvers.stack_output:StackOutputExternal", + "no_value = sceptre.resolvers.no_value:NoValue", + "select = sceptre.resolvers.select:Select", + "stack_attr = sceptre.resolvers.stack_attr:StackAttr", + "sub = sceptre.resolvers.sub:Sub", + "split = sceptre.resolvers.split:Split", + "join = sceptre.resolvers.join:Join", + ], + "sceptre.template_handlers": [ + "file = sceptre.template_handlers.file:File", + "s3 = sceptre.template_handlers.s3:S3", + "http = sceptre.template_handlers.http:Http", + ], }, data_files=[ - (path.join("sceptre", "stack_policies"), [ - path.join("sceptre", "stack_policies", "lock.json"), - path.join("sceptre", "stack_policies", "unlock.json") - ]) + ( + os.path.join("sceptre", "stack_policies"), + [ + os.path.join("sceptre", "stack_policies", "lock.json"), + os.path.join("sceptre", "stack_policies", "unlock.json"), + ], + ) ], include_package_data=True, zip_safe=False, @@ -83,16 +100,12 @@ "Intended Audience :: Developers", "Natural Language :: English", "Environment :: Console", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7" + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], test_suite="tests", install_requires=install_requirements, - tests_require=test_requirements, - setup_requires=setup_requirements, - extras_require={ - "test": test_requirements - } + extras_require=extra_requirements, ) diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index bbc70967b..000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,2 +0,0 @@ -sonar.coverage.exclusions=**/__pycache__,tests/**,coverage-reports/**,integration-tests/**,docs/**,setup.py -sonar.exclusions=**/__pycache__,coverage-reports/**,docs/**,tests/fixtures** diff --git a/sponsors/cloudreach_logo.png b/sponsors/cloudreach_logo.png new file mode 100644 index 000000000..eec735c59 Binary files /dev/null and b/sponsors/cloudreach_logo.png differ diff --git a/sponsors/godaddy_logo.png b/sponsors/godaddy_logo.png new file mode 100644 index 000000000..24e970ada Binary files /dev/null and b/sponsors/godaddy_logo.png differ diff --git a/sponsors/sage_bionetworks_logo.png b/sponsors/sage_bionetworks_logo.png new file mode 100644 index 000000000..661a6a24c Binary files /dev/null and b/sponsors/sage_bionetworks_logo.png differ diff --git a/tests/fixtures-vpc/config/account/stack-group/region/vpc.yaml b/tests/fixtures-vpc/config/account/stack-group/region/vpc.yaml index 97aa110ed..889e79ded 100644 --- a/tests/fixtures-vpc/config/account/stack-group/region/vpc.yaml +++ b/tests/fixtures-vpc/config/account/stack-group/region/vpc.yaml @@ -1,4 +1,5 @@ -template_path: path/to/template +template: + path: path/to/template parameters: param1: val1 dependencies: diff --git a/tests/fixtures-vpc/config/top/level.yaml b/tests/fixtures-vpc/config/top/level.yaml index 460977c19..0457dab1e 100644 --- a/tests/fixtures-vpc/config/top/level.yaml +++ b/tests/fixtures-vpc/config/top/level.yaml @@ -1 +1,2 @@ -template_path: somethingelse.py +template: + path: somethingelse.py diff --git a/tests/fixtures-vpc/hooks/custom_hook.py b/tests/fixtures-vpc/hooks/custom_hook.py index af631231d..0e054117f 100644 --- a/tests/fixtures-vpc/hooks/custom_hook.py +++ b/tests/fixtures-vpc/hooks/custom_hook.py @@ -8,6 +8,7 @@ class CustomHook(Hook): This is a test task. """ + def __init__(self, *args, **kwargs): super(CustomHook, self).__init__(*args, **kwargs) diff --git a/tests/fixtures-vpc/templates/compiled_vpc.json b/tests/fixtures-vpc/templates/compiled_vpc.json index aecc6cc2e..e6ac5dce6 100644 --- a/tests/fixtures-vpc/templates/compiled_vpc.json +++ b/tests/fixtures-vpc/templates/compiled_vpc.json @@ -33,8 +33,8 @@ "CidrBlock": { "Ref": "CidrBlock" }, - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, "InstanceTenancy": "default" }, "Type": "AWS::EC2::VPC" diff --git a/tests/fixtures-vpc/templates/compiled_vpc_sud.json b/tests/fixtures-vpc/templates/compiled_vpc_sud.json index 5551d3ad7..6a03e9739 100644 --- a/tests/fixtures-vpc/templates/compiled_vpc_sud.json +++ b/tests/fixtures-vpc/templates/compiled_vpc_sud.json @@ -25,8 +25,8 @@ "VirtualPrivateCloud": { "Properties": { "CidrBlock": "10.0.0.0/16", - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, "InstanceTenancy": "default" }, "Type": "AWS::EC2::VPC" diff --git a/tests/fixtures-vpc/templates/vpc.json b/tests/fixtures-vpc/templates/vpc.json index aecc6cc2e..e6ac5dce6 100644 --- a/tests/fixtures-vpc/templates/vpc.json +++ b/tests/fixtures-vpc/templates/vpc.json @@ -33,8 +33,8 @@ "CidrBlock": { "Ref": "CidrBlock" }, - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, "InstanceTenancy": "default" }, "Type": "AWS::EC2::VPC" diff --git a/tests/fixtures-vpc/templates/vpc.py b/tests/fixtures-vpc/templates/vpc.py index 812c46877..3f51ce506 100644 --- a/tests/fixtures-vpc/templates/vpc.py +++ b/tests/fixtures-vpc/templates/vpc.py @@ -8,33 +8,37 @@ def sceptre_handler(sceptre_user_data): t = Template() - cidr_block_param = t.add_parameter(Parameter( - "CidrBlock", - Type="String", - Default="10.0.0.0/16", - )) - - vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=Ref(cidr_block_param), - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, - )) - - igw = t.add_resource(InternetGateway( - "InternetGateway", - )) - - t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(vpc), - InternetGatewayId=Ref(igw), - )) - - t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(vpc) - )) + cidr_block_param = t.add_parameter( + Parameter( + "CidrBlock", + Type="String", + Default="10.0.0.0/16", + ) + ) + + vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=Ref(cidr_block_param), + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) + ) + + igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) + ) + + t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(vpc), + InternetGatewayId=Ref(igw), + ) + ) + + t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(vpc))) return t.to_json() diff --git a/tests/fixtures-vpc/templates/vpc.template b/tests/fixtures-vpc/templates/vpc.template index aecc6cc2e..e6ac5dce6 100644 --- a/tests/fixtures-vpc/templates/vpc.template +++ b/tests/fixtures-vpc/templates/vpc.template @@ -33,8 +33,8 @@ "CidrBlock": { "Ref": "CidrBlock" }, - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, "InstanceTenancy": "default" }, "Type": "AWS::EC2::VPC" diff --git a/tests/fixtures-vpc/templates/vpc_sgt.py b/tests/fixtures-vpc/templates/vpc_sgt.py index e5c319469..dca58ca6b 100644 --- a/tests/fixtures-vpc/templates/vpc_sgt.py +++ b/tests/fixtures-vpc/templates/vpc_sgt.py @@ -6,7 +6,6 @@ class VpcTemplate(object): - def __init__(self): self.template = Template() @@ -20,44 +19,48 @@ def __init__(self): def add_parameters(self): t = self.template - self.cidr_block_param = t.add_parameter(Parameter( - "CidrBlock", - Type="String", - Default="10.0.0.0/16", - )) + self.cidr_block_param = t.add_parameter( + Parameter( + "CidrBlock", + Type="String", + Default="10.0.0.0/16", + ) + ) def add_vpc(self): t = self.template - self.vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=Ref(self.cidr_block_param), - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, - )) + self.vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=Ref(self.cidr_block_param), + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) + ) def add_igw(self): t = self.template - self.igw = t.add_resource(InternetGateway( - "InternetGateway", - )) + self.igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) + ) - t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(self.vpc), - InternetGatewayId=Ref(self.igw), - )) + t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(self.vpc), + InternetGatewayId=Ref(self.igw), + ) + ) def add_outputs(self): t = self.template - t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(self.vpc) - )) + t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(self.vpc))) def sceptre_handler(sceptre_user_data): diff --git a/tests/fixtures-vpc/templates/vpc_sud.py b/tests/fixtures-vpc/templates/vpc_sud.py index 52a3143d4..0ec2b2fc6 100644 --- a/tests/fixtures-vpc/templates/vpc_sud.py +++ b/tests/fixtures-vpc/templates/vpc_sud.py @@ -6,31 +6,32 @@ def sceptre_handler(sceptre_user_data): - t = Template() - vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=sceptre_user_data["cidr_block"], - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, - )) - - igw = t.add_resource(InternetGateway( - "InternetGateway", - )) - - t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(vpc), - InternetGatewayId=Ref(igw), - )) - - t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(vpc) - )) + vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=sceptre_user_data["cidr_block"], + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) + ) + + igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) + ) + + t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(vpc), + InternetGatewayId=Ref(igw), + ) + ) + + t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(vpc))) return t.to_json() diff --git a/tests/fixtures-vpc/templates/vpc_t.py b/tests/fixtures-vpc/templates/vpc_t.py index 1f9ae8ffe..8f1ee2d55 100644 --- a/tests/fixtures-vpc/templates/vpc_t.py +++ b/tests/fixtures-vpc/templates/vpc_t.py @@ -6,32 +6,36 @@ t = Template() -cidr_block_param = t.add_parameter(Parameter( - "CidrBlock", - Type="String", - Default="10.0.0.0/16", -)) - -vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=Ref(cidr_block_param), - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, -)) - -igw = t.add_resource(InternetGateway( - "InternetGateway", -)) - -igw_attachment = t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(vpc), - InternetGatewayId=Ref(igw), -)) - -vpc_id_output = t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(vpc) -)) +cidr_block_param = t.add_parameter( + Parameter( + "CidrBlock", + Type="String", + Default="10.0.0.0/16", + ) +) + +vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=Ref(cidr_block_param), + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) +) + +igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) +) + +igw_attachment = t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(vpc), + InternetGatewayId=Ref(igw), + ) +) + +vpc_id_output = t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(vpc))) diff --git a/tests/fixtures/config/account/stack-group/region/construct_nodes.yaml b/tests/fixtures/config/account/stack-group/region/construct_nodes.yaml index 1c06705ec..928a89402 100644 --- a/tests/fixtures/config/account/stack-group/region/construct_nodes.yaml +++ b/tests/fixtures/config/account/stack-group/region/construct_nodes.yaml @@ -1,4 +1,5 @@ -template_path: path/to/template +template: + path: path/to/template parameters: param1: !environment_variable example param2: diff --git a/tests/fixtures/hooks/custom_hook.py b/tests/fixtures/hooks/custom_hook.py index af631231d..0e054117f 100644 --- a/tests/fixtures/hooks/custom_hook.py +++ b/tests/fixtures/hooks/custom_hook.py @@ -8,6 +8,7 @@ class CustomHook(Hook): This is a test task. """ + def __init__(self, *args, **kwargs): super(CustomHook, self).__init__(*args, **kwargs) diff --git a/tests/fixtures/templates/chdir.py b/tests/fixtures/templates/chdir.py new file mode 100644 index 000000000..9d3dfca1c --- /dev/null +++ b/tests/fixtures/templates/chdir.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from troposphere import Template + +from os import chdir, getcwd + + +def sceptre_handler(sceptre_user_data): + t = Template() + + curr_dir = getcwd() + chdir("..") + chdir(curr_dir) + + return t.to_json() diff --git a/tests/fixtures/templates/compiled_vpc.json b/tests/fixtures/templates/compiled_vpc.json index aecc6cc2e..e6ac5dce6 100644 --- a/tests/fixtures/templates/compiled_vpc.json +++ b/tests/fixtures/templates/compiled_vpc.json @@ -33,8 +33,8 @@ "CidrBlock": { "Ref": "CidrBlock" }, - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, "InstanceTenancy": "default" }, "Type": "AWS::EC2::VPC" diff --git a/tests/fixtures/templates/compiled_vpc.yaml b/tests/fixtures/templates/compiled_vpc.yaml new file mode 100644 index 000000000..a96fc1dd7 --- /dev/null +++ b/tests/fixtures/templates/compiled_vpc.yaml @@ -0,0 +1,10 @@ +--- +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 +Outputs: + VpcId: + Value: + Ref: VPC diff --git a/tests/fixtures/templates/compiled_vpc_sud.json b/tests/fixtures/templates/compiled_vpc_sud.json index 5551d3ad7..6a03e9739 100644 --- a/tests/fixtures/templates/compiled_vpc_sud.json +++ b/tests/fixtures/templates/compiled_vpc_sud.json @@ -25,8 +25,8 @@ "VirtualPrivateCloud": { "Properties": { "CidrBlock": "10.0.0.0/16", - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, "InstanceTenancy": "default" }, "Type": "AWS::EC2::VPC" diff --git a/tests/fixtures/templates/vpc.json b/tests/fixtures/templates/vpc.json deleted file mode 100644 index aecc6cc2e..000000000 --- a/tests/fixtures/templates/vpc.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "Outputs": { - "VpcId": { - "Description": "New VPC ID", - "Value": { - "Ref": "VirtualPrivateCloud" - } - } - }, - "Parameters": { - "CidrBlock": { - "Default": "10.0.0.0/16", - "Type": "String" - } - }, - "Resources": { - "IGWAttachment": { - "Properties": { - "InternetGatewayId": { - "Ref": "InternetGateway" - }, - "VpcId": { - "Ref": "VirtualPrivateCloud" - } - }, - "Type": "AWS::EC2::VPCGatewayAttachment" - }, - "InternetGateway": { - "Type": "AWS::EC2::InternetGateway" - }, - "VirtualPrivateCloud": { - "Properties": { - "CidrBlock": { - "Ref": "CidrBlock" - }, - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", - "InstanceTenancy": "default" - }, - "Type": "AWS::EC2::VPC" - } - } -} diff --git a/tests/fixtures/templates/vpc.py b/tests/fixtures/templates/vpc.py index 812c46877..3f51ce506 100644 --- a/tests/fixtures/templates/vpc.py +++ b/tests/fixtures/templates/vpc.py @@ -8,33 +8,37 @@ def sceptre_handler(sceptre_user_data): t = Template() - cidr_block_param = t.add_parameter(Parameter( - "CidrBlock", - Type="String", - Default="10.0.0.0/16", - )) - - vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=Ref(cidr_block_param), - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, - )) - - igw = t.add_resource(InternetGateway( - "InternetGateway", - )) - - t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(vpc), - InternetGatewayId=Ref(igw), - )) - - t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(vpc) - )) + cidr_block_param = t.add_parameter( + Parameter( + "CidrBlock", + Type="String", + Default="10.0.0.0/16", + ) + ) + + vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=Ref(cidr_block_param), + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) + ) + + igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) + ) + + t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(vpc), + InternetGatewayId=Ref(igw), + ) + ) + + t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(vpc))) return t.to_json() diff --git a/tests/fixtures/templates/vpc.template b/tests/fixtures/templates/vpc.template index aecc6cc2e..e6ac5dce6 100644 --- a/tests/fixtures/templates/vpc.template +++ b/tests/fixtures/templates/vpc.template @@ -33,8 +33,8 @@ "CidrBlock": { "Ref": "CidrBlock" }, - "EnableDnsHostnames": "true", - "EnableDnsSupport": "true", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, "InstanceTenancy": "default" }, "Type": "AWS::EC2::VPC" diff --git a/tests/fixtures/templates/vpc.without_start_marker.yaml b/tests/fixtures/templates/vpc.without_start_marker.yaml new file mode 100644 index 000000000..effdeaa05 --- /dev/null +++ b/tests/fixtures/templates/vpc.without_start_marker.yaml @@ -0,0 +1,27 @@ +Outputs: + VpcId: + Description: New VPC ID + Value: + Ref: VirtualPrivateCloud +Parameters: + CidrBlock: + Default: 10.0.0.0/16 + Type: String +Resources: + IGWAttachment: + Properties: + InternetGatewayId: + Ref: InternetGateway + VpcId: + Ref: VirtualPrivateCloud + Type: AWS::EC2::VPCGatewayAttachment + InternetGateway: + Type: AWS::EC2::InternetGateway + VirtualPrivateCloud: + Properties: + CidrBlock: + Ref: CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + Type: AWS::EC2::VPC diff --git a/tests/fixtures/templates/vpc.yaml b/tests/fixtures/templates/vpc.yaml index 3355df367..92c6a39c0 100644 --- a/tests/fixtures/templates/vpc.yaml +++ b/tests/fixtures/templates/vpc.yaml @@ -22,7 +22,7 @@ Resources: Properties: CidrBlock: Ref: CidrBlock - EnableDnsHostnames: 'true' - EnableDnsSupport: 'true' + EnableDnsHostnames: true + EnableDnsSupport: true InstanceTenancy: default Type: AWS::EC2::VPC diff --git a/tests/fixtures/templates/vpc_sgt.py b/tests/fixtures/templates/vpc_sgt.py index e5c319469..dca58ca6b 100644 --- a/tests/fixtures/templates/vpc_sgt.py +++ b/tests/fixtures/templates/vpc_sgt.py @@ -6,7 +6,6 @@ class VpcTemplate(object): - def __init__(self): self.template = Template() @@ -20,44 +19,48 @@ def __init__(self): def add_parameters(self): t = self.template - self.cidr_block_param = t.add_parameter(Parameter( - "CidrBlock", - Type="String", - Default="10.0.0.0/16", - )) + self.cidr_block_param = t.add_parameter( + Parameter( + "CidrBlock", + Type="String", + Default="10.0.0.0/16", + ) + ) def add_vpc(self): t = self.template - self.vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=Ref(self.cidr_block_param), - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, - )) + self.vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=Ref(self.cidr_block_param), + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) + ) def add_igw(self): t = self.template - self.igw = t.add_resource(InternetGateway( - "InternetGateway", - )) + self.igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) + ) - t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(self.vpc), - InternetGatewayId=Ref(self.igw), - )) + t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(self.vpc), + InternetGatewayId=Ref(self.igw), + ) + ) def add_outputs(self): t = self.template - t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(self.vpc) - )) + t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(self.vpc))) def sceptre_handler(sceptre_user_data): diff --git a/tests/fixtures/templates/vpc_sud.py b/tests/fixtures/templates/vpc_sud.py index 52a3143d4..0ec2b2fc6 100644 --- a/tests/fixtures/templates/vpc_sud.py +++ b/tests/fixtures/templates/vpc_sud.py @@ -6,31 +6,32 @@ def sceptre_handler(sceptre_user_data): - t = Template() - vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=sceptre_user_data["cidr_block"], - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, - )) - - igw = t.add_resource(InternetGateway( - "InternetGateway", - )) - - t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(vpc), - InternetGatewayId=Ref(igw), - )) - - t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(vpc) - )) + vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=sceptre_user_data["cidr_block"], + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) + ) + + igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) + ) + + t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(vpc), + InternetGatewayId=Ref(igw), + ) + ) + + t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(vpc))) return t.to_json() diff --git a/tests/fixtures/templates/vpc_t.py b/tests/fixtures/templates/vpc_t.py index 1f9ae8ffe..8f1ee2d55 100644 --- a/tests/fixtures/templates/vpc_t.py +++ b/tests/fixtures/templates/vpc_t.py @@ -6,32 +6,36 @@ t = Template() -cidr_block_param = t.add_parameter(Parameter( - "CidrBlock", - Type="String", - Default="10.0.0.0/16", -)) - -vpc = t.add_resource(VPC( - "VirtualPrivateCloud", - CidrBlock=Ref(cidr_block_param), - InstanceTenancy="default", - EnableDnsSupport=True, - EnableDnsHostnames=True, -)) - -igw = t.add_resource(InternetGateway( - "InternetGateway", -)) - -igw_attachment = t.add_resource(VPCGatewayAttachment( - "IGWAttachment", - VpcId=Ref(vpc), - InternetGatewayId=Ref(igw), -)) - -vpc_id_output = t.add_output(Output( - "VpcId", - Description="New VPC ID", - Value=Ref(vpc) -)) +cidr_block_param = t.add_parameter( + Parameter( + "CidrBlock", + Type="String", + Default="10.0.0.0/16", + ) +) + +vpc = t.add_resource( + VPC( + "VirtualPrivateCloud", + CidrBlock=Ref(cidr_block_param), + InstanceTenancy="default", + EnableDnsSupport=True, + EnableDnsHostnames=True, + ) +) + +igw = t.add_resource( + InternetGateway( + "InternetGateway", + ) +) + +igw_attachment = t.add_resource( + VPCGatewayAttachment( + "IGWAttachment", + VpcId=Ref(vpc), + InternetGatewayId=Ref(igw), + ) +) + +vpc_id_output = t.add_output(Output("VpcId", Description="New VPC ID", Value=Ref(vpc))) diff --git a/tests/test_actions.py b/tests/test_actions.py index c6c689f5b..7bc4e5514 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -1,48 +1,66 @@ # -*- coding: utf-8 -*- - -import pytest -from mock import patch, sentinel, Mock, call - import datetime -from dateutil.tz import tzutc +import json +from unittest.mock import patch, sentinel, Mock, call, ANY +import pytest from botocore.exceptions import ClientError +from dateutil.tz import tzutc -from sceptre.stack import Stack +from sceptre.exceptions import ( + CannotUpdateFailedStackError, + ProtectedStackError, + StackDoesNotExistError, + UnknownStackChangeSetStatusError, + UnknownStackStatusError, +) from sceptre.plan.actions import StackActions +from sceptre.stack import Stack +from sceptre.stack_status import StackChangeSetStatus, StackStatus from sceptre.template import Template -from sceptre.stack_status import StackStatus -from sceptre.stack_status import StackChangeSetStatus -from sceptre.exceptions import CannotUpdateFailedStackError -from sceptre.exceptions import UnknownStackStatusError -from sceptre.exceptions import UnknownStackChangeSetStatusError -from sceptre.exceptions import StackDoesNotExistError -from sceptre.exceptions import ProtectedStackError class TestStackActions(object): - def setup_method(self, test_method): self.patcher_connection_manager = patch( "sceptre.plan.actions.ConnectionManager" ) self.mock_ConnectionManager = self.patcher_connection_manager.start() self.stack = Stack( - name='prod/app/stack', project_code=sentinel.project_code, - template_path=sentinel.template_path, region=sentinel.region, - profile=sentinel.profile, parameters={"key1": "val1"}, - sceptre_user_data=sentinel.sceptre_user_data, hooks={}, - s3_details=None, dependencies=sentinel.dependencies, - role_arn=sentinel.role_arn, protected=False, - tags={"tag1": "val1"}, external_name=sentinel.external_name, + name="prod/app/stack", + project_code=sentinel.project_code, + template_path=sentinel.template_path, + region=sentinel.region, + profile=sentinel.profile, + parameters={"key1": "val1"}, + sceptre_user_data=sentinel.sceptre_user_data, + hooks={}, + s3_details=None, + dependencies=sentinel.dependencies, + cloudformation_service_role=sentinel.cloudformation_service_role, + protected=False, + tags={"tag1": "val1"}, + external_name=sentinel.external_name, notifications=[sentinel.notification], on_failure=sentinel.on_failure, - stack_timeout=sentinel.stack_timeout + disable_rollback=False, + stack_timeout=sentinel.stack_timeout, ) self.actions = StackActions(self.stack) + self.stack_group_config = {} self.template = Template( - "fixtures/templates", self.stack.sceptre_user_data, - self.actions.connection_manager, self.stack.s3_details + "fixtures/templates", + self.stack.template_handler_config, + self.stack.sceptre_user_data, + self.stack_group_config, + self.actions.connection_manager, + self.stack.s3_details, + ) + self.template._body = json.dumps( + { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": {"Bucket": {"Type": "AWS::S3::Bucket", "Properties": {}}}, + } ) self.stack._template = self.template @@ -56,10 +74,12 @@ def test_template_loads_template(self, mock_Template): response = self.stack.template mock_Template.assert_called_once_with( - path=sentinel.template_path, + name="prod/app/stack", + handler_config={"type": "file", "path": sentinel.template_path}, sceptre_user_data=sentinel.sceptre_user_data, + stack_group_config={}, connection_manager=self.stack.connection_manager, - s3_details=None + s3_details=None, ) assert response == sentinel.template @@ -70,9 +90,11 @@ def test_template_returns_template_if_it_exists(self): def test_external_name_with_custom_stack_name(self): stack = Stack( - name="stack_name", project_code="project_code", - template_path="template_path", region="region", - external_name="external_name" + name="stack_name", + project_code="project_code", + template_path="template_path", + region="region", + external_name="external_name", ) assert stack.external_name == "external_name" @@ -80,13 +102,11 @@ def test_external_name_with_custom_stack_name(self): @patch("sceptre.plan.actions.StackActions._wait_for_completion") @patch("sceptre.plan.actions.StackActions._get_stack_timeout") def test_create_sends_correct_request( - self, mock_get_stack_timeout, mock_wait_for_completion + self, mock_get_stack_timeout, mock_wait_for_completion ): self.template._body = sentinel.template - mock_get_stack_timeout.return_value = { - "TimeoutInMinutes": sentinel.timeout - } + mock_get_stack_timeout.return_value = {"TimeoutInMinutes": sentinel.timeout} self.actions.create() self.actions.connection_manager.call.assert_called_with( @@ -95,23 +115,53 @@ def test_create_sends_correct_request( kwargs={ "StackName": sentinel.external_name, "TemplateBody": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], - "RoleARN": sentinel.role_arn, - "NotificationARNs": [sentinel.notification], - "Tags": [ - {"Key": "tag1", "Value": "val1"} + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", ], + "RoleARN": sentinel.cloudformation_service_role, + "NotificationARNs": [sentinel.notification], + "Tags": [{"Key": "tag1", "Value": "val1"}], "OnFailure": sentinel.on_failure, - "TimeoutInMinutes": sentinel.timeout - } + "TimeoutInMinutes": sentinel.timeout, + }, + ) + mock_wait_for_completion.assert_called_once_with(boto_response=ANY) + + @patch("sceptre.plan.actions.StackActions._wait_for_completion") + @patch("sceptre.plan.actions.StackActions._get_stack_timeout") + def test_create_disable_rollback_overrides_on_failure( + self, mock_get_stack_timeout, mock_wait_for_completion + ): + self.template._body = sentinel.template + self.actions.stack.on_failure = "ROLLBACK" + self.actions.stack.disable_rollback = True + + mock_get_stack_timeout.return_value = {"TimeoutInMinutes": sentinel.timeout} + + self.actions.create() + self.actions.connection_manager.call.assert_called_with( + service="cloudformation", + command="create_stack", + kwargs={ + "StackName": sentinel.external_name, + "TemplateBody": sentinel.template, + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], + "RoleARN": sentinel.cloudformation_service_role, + "NotificationARNs": [sentinel.notification], + "Tags": [{"Key": "tag1", "Value": "val1"}], + "DisableRollback": True, + "TimeoutInMinutes": sentinel.timeout, + }, ) - mock_wait_for_completion.assert_called_once_with() + mock_wait_for_completion.assert_called_once_with(boto_response=ANY) @patch("sceptre.plan.actions.StackActions._wait_for_completion") def test_create_sends_correct_request_no_notifications( @@ -130,23 +180,20 @@ def test_create_sends_correct_request_no_notifications( kwargs={ "StackName": sentinel.external_name, "Template": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], - "RoleARN": sentinel.role_arn, - "NotificationARNs": [], - "Tags": [ - {"Key": "tag1", "Value": "val1"} + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", ], + "RoleARN": sentinel.cloudformation_service_role, + "NotificationARNs": [], + "Tags": [{"Key": "tag1", "Value": "val1"}], "OnFailure": sentinel.on_failure, - "TimeoutInMinutes": sentinel.stack_timeout - } + "TimeoutInMinutes": sentinel.stack_timeout, + }, ) - mock_wait_for_completion.assert_called_once_with() + mock_wait_for_completion.assert_called_once_with(boto_response=ANY) @patch("sceptre.plan.actions.StackActions._wait_for_completion") def test_create_sends_correct_request_with_no_failure_no_timeout( @@ -164,21 +211,39 @@ def test_create_sends_correct_request_with_no_failure_no_timeout( kwargs={ "StackName": sentinel.external_name, "TemplateBody": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], - "RoleARN": sentinel.role_arn, + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], + "RoleARN": sentinel.cloudformation_service_role, "NotificationARNs": [sentinel.notification], - "Tags": [ - {"Key": "tag1", "Value": "val1"} - ] - } + "Tags": [{"Key": "tag1", "Value": "val1"}], + }, ) - mock_wait_for_completion.assert_called_once_with() + + mock_wait_for_completion.assert_called_once_with(boto_response=ANY) + + @patch("sceptre.plan.actions.StackActions._wait_for_completion") + def test_create_stack_already_exists(self, mock_wait_for_completion): + self.actions.stack._template = Mock(spec=Template) + self.actions.stack._template.get_boto_call_parameter.return_value = { + "Template": sentinel.template + } + mock_wait_for_completion.side_effect = ClientError( + { + "Error": { + "Code": "AlreadyExistsException", + "Message": "Stack already [{}] exists".format( + self.actions.stack.name + ), + } + }, + sentinel.operation, + ) + response = self.actions.create() + assert response == StackStatus.COMPLETE @patch("sceptre.plan.actions.StackActions._wait_for_completion") def test_update_sends_correct_request(self, mock_wait_for_completion): @@ -194,22 +259,19 @@ def test_update_sends_correct_request(self, mock_wait_for_completion): kwargs={ "StackName": sentinel.external_name, "Template": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], - "RoleARN": sentinel.role_arn, + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], + "RoleARN": sentinel.cloudformation_service_role, "NotificationARNs": [sentinel.notification], - "Tags": [ - {"Key": "tag1", "Value": "val1"} - ] - } + "Tags": [{"Key": "tag1", "Value": "val1"}], + }, ) mock_wait_for_completion.assert_called_once_with( - sentinel.stack_timeout + sentinel.stack_timeout, boto_response=ANY ) @patch("sceptre.plan.actions.StackActions._wait_for_completion") @@ -228,32 +290,31 @@ def test_update_cancels_after_timeout(self, mock_wait_for_completion): kwargs={ "StackName": sentinel.external_name, "Template": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], - "RoleARN": sentinel.role_arn, + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], + "RoleARN": sentinel.cloudformation_service_role, "NotificationARNs": [sentinel.notification], - "Tags": [ - {"Key": "tag1", "Value": "val1"} - ] - }), + "Tags": [{"Key": "tag1", "Value": "val1"}], + }, + ), call( service="cloudformation", command="cancel_update_stack", - kwargs={"StackName": sentinel.external_name}) + kwargs={"StackName": sentinel.external_name}, + ), ] self.actions.connection_manager.call.assert_has_calls(calls) mock_wait_for_completion.assert_has_calls( - [call(sentinel.stack_timeout), call()] + [call(sentinel.stack_timeout, boto_response=ANY), call(boto_response=ANY)] ) @patch("sceptre.plan.actions.StackActions._wait_for_completion") def test_update_sends_correct_request_no_notification( - self, mock_wait_for_completion + self, mock_wait_for_completion ): self.actions.stack._template = Mock(spec=Template) self.actions.stack._template.get_boto_call_parameter.return_value = { @@ -268,27 +329,24 @@ def test_update_sends_correct_request_no_notification( kwargs={ "StackName": sentinel.external_name, "Template": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], - "RoleARN": sentinel.role_arn, + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], + "RoleARN": sentinel.cloudformation_service_role, "NotificationARNs": [], - "Tags": [ - {"Key": "tag1", "Value": "val1"} - ] - } + "Tags": [{"Key": "tag1", "Value": "val1"}], + }, ) mock_wait_for_completion.assert_called_once_with( - sentinel.stack_timeout + sentinel.stack_timeout, boto_response=ANY ) @patch("sceptre.plan.actions.StackActions._wait_for_completion") def test_update_with_complete_stack_with_no_updates_to_perform( - self, mock_wait_for_completion + self, mock_wait_for_completion ): self.actions.stack._template = Mock(spec=Template) self.actions.stack._template.get_boto_call_parameter.return_value = { @@ -298,31 +356,27 @@ def test_update_with_complete_stack_with_no_updates_to_perform( { "Error": { "Code": "NoUpdateToPerformError", - "Message": "No updates are to be performed." + "Message": "No updates are to be performed.", } }, - sentinel.operation + sentinel.operation, ) response = self.actions.update() assert response == StackStatus.COMPLETE @patch("sceptre.plan.actions.StackActions._wait_for_completion") - def test_cancel_update_sends_correct_request( - self, mock_wait_for_completion - ): + def test_cancel_update_sends_correct_request(self, mock_wait_for_completion): self.actions.cancel_stack_update() self.actions.connection_manager.call.assert_called_once_with( service="cloudformation", command="cancel_update_stack", - kwargs={"StackName": sentinel.external_name} + kwargs={"StackName": sentinel.external_name}, ) - mock_wait_for_completion.assert_called_once_with() + mock_wait_for_completion.assert_called_once_with(boto_response=ANY) @patch("sceptre.plan.actions.StackActions.create") @patch("sceptre.plan.actions.StackActions._get_status") - def test_launch_with_stack_that_does_not_exist( - self, mock_get_status, mock_create - ): + def test_launch_with_stack_that_does_not_exist(self, mock_get_status, mock_create): mock_get_status.side_effect = StackDoesNotExistError() mock_create.return_value = sentinel.launch_response response = self.actions.launch() @@ -333,7 +387,7 @@ def test_launch_with_stack_that_does_not_exist( @patch("sceptre.plan.actions.StackActions.delete") @patch("sceptre.plan.actions.StackActions._get_status") def test_launch_with_stack_that_failed_to_create( - self, mock_get_status, mock_delete, mock_create + self, mock_get_status, mock_delete, mock_create ): mock_get_status.return_value = "CREATE_FAILED" mock_create.return_value = sentinel.launch_response @@ -345,7 +399,7 @@ def test_launch_with_stack_that_failed_to_create( @patch("sceptre.plan.actions.StackActions.update") @patch("sceptre.plan.actions.StackActions._get_status") def test_launch_with_complete_stack_with_updates_to_perform( - self, mock_get_status, mock_update + self, mock_get_status, mock_update ): mock_get_status.return_value = "CREATE_COMPLETE" mock_update.return_value = sentinel.launch_response @@ -356,7 +410,7 @@ def test_launch_with_complete_stack_with_updates_to_perform( @patch("sceptre.plan.actions.StackActions.update") @patch("sceptre.plan.actions.StackActions._get_status") def test_launch_with_complete_stack_with_no_updates_to_perform( - self, mock_get_status, mock_update + self, mock_get_status, mock_update ): mock_get_status.return_value = "CREATE_COMPLETE" mock_update.return_value = StackStatus.COMPLETE @@ -367,17 +421,11 @@ def test_launch_with_complete_stack_with_no_updates_to_perform( @patch("sceptre.plan.actions.StackActions.update") @patch("sceptre.plan.actions.StackActions._get_status") def test_launch_with_complete_stack_with_unknown_client_error( - self, mock_get_status, mock_update + self, mock_get_status, mock_update ): mock_get_status.return_value = "CREATE_COMPLETE" mock_update.side_effect = ClientError( - { - "Error": { - "Code": "Boom!", - "Message": "Boom!" - } - }, - sentinel.operation + {"Error": {"Code": "Boom!", "Message": "Boom!"}}, sentinel.operation ) with pytest.raises(ClientError): self.actions.launch() @@ -403,9 +451,7 @@ def test_launch_with_unknown_stack_status(self, mock_get_status): @patch("sceptre.plan.actions.StackActions._wait_for_completion") @patch("sceptre.plan.actions.StackActions._get_status") - def test_delete_with_created_stack( - self, mock_get_status, mock_wait_for_completion - ): + def test_delete_with_created_stack(self, mock_get_status, mock_wait_for_completion): mock_get_status.return_value = "CREATE_COMPLETE" self.actions.delete() @@ -414,14 +460,14 @@ def test_delete_with_created_stack( command="delete_stack", kwargs={ "StackName": sentinel.external_name, - "RoleARN": sentinel.role_arn - } + "RoleARN": sentinel.cloudformation_service_role, + }, ) @patch("sceptre.plan.actions.StackActions._wait_for_completion") @patch("sceptre.plan.actions.StackActions._get_status") def test_delete_when_wait_for_completion_raises_stack_does_not_exist_error( - self, mock_get_status, mock_wait_for_completion + self, mock_get_status, mock_wait_for_completion ): mock_get_status.return_value = "CREATE_COMPLETE" mock_wait_for_completion.side_effect = StackDoesNotExistError() @@ -431,17 +477,17 @@ def test_delete_when_wait_for_completion_raises_stack_does_not_exist_error( @patch("sceptre.plan.actions.StackActions._wait_for_completion") @patch("sceptre.plan.actions.StackActions._get_status") def test_delete_when_wait_for_completion_raises_non_existent_client_error( - self, mock_get_status, mock_wait_for_completion + self, mock_get_status, mock_wait_for_completion ): mock_get_status.return_value = "CREATE_COMPLETE" mock_wait_for_completion.side_effect = ClientError( { "Error": { "Code": "DoesNotExistException", - "Message": "Stack does not exist" + "Message": "Stack does not exist", } }, - sentinel.operation + sentinel.operation, ) status = self.actions.delete() assert status == StackStatus.COMPLETE @@ -449,17 +495,12 @@ def test_delete_when_wait_for_completion_raises_non_existent_client_error( @patch("sceptre.plan.actions.StackActions._wait_for_completion") @patch("sceptre.plan.actions.StackActions._get_status") def test_delete_when_wait_for_completion_raises_unexpected_client_error( - self, mock_get_status, mock_wait_for_completion + self, mock_get_status, mock_wait_for_completion ): mock_get_status.return_value = "CREATE_COMPLETE" mock_wait_for_completion.side_effect = ClientError( - { - "Error": { - "Code": "DoesNotExistException", - "Message": "Boom" - } - }, - sentinel.operation + {"Error": {"Code": "DoesNotExistException", "Message": "Boom"}}, + sentinel.operation, ) with pytest.raises(ClientError): self.actions.delete() @@ -467,7 +508,7 @@ def test_delete_when_wait_for_completion_raises_unexpected_client_error( @patch("sceptre.plan.actions.StackActions._wait_for_completion") @patch("sceptre.plan.actions.StackActions._get_status") def test_delete_with_non_existent_stack( - self, mock_get_status, mock_wait_for_completion + self, mock_get_status, mock_wait_for_completion ): mock_get_status.side_effect = StackDoesNotExistError() status = self.actions.delete() @@ -478,7 +519,7 @@ def test_describe_stack_sends_correct_request(self): self.actions.connection_manager.call.assert_called_with( service="cloudformation", command="describe_stacks", - kwargs={"StackName": sentinel.external_name} + kwargs={"StackName": sentinel.external_name}, ) def test_describe_events_sends_correct_request(self): @@ -486,7 +527,7 @@ def test_describe_events_sends_correct_request(self): self.actions.connection_manager.call.assert_called_with( service="cloudformation", command="describe_stack_events", - kwargs={"StackName": sentinel.external_name} + kwargs={"StackName": sentinel.external_name}, ) def test_describe_resources_sends_correct_request(self): @@ -495,7 +536,7 @@ def test_describe_resources_sends_correct_request(self): { "LogicalResourceId": sentinel.logical_resource_id, "PhysicalResourceId": sentinel.physical_resource_id, - "OtherParam": sentinel.other_param + "OtherParam": sentinel.other_param, } ] } @@ -503,33 +544,27 @@ def test_describe_resources_sends_correct_request(self): self.actions.connection_manager.call.assert_called_with( service="cloudformation", command="describe_stack_resources", - kwargs={"StackName": sentinel.external_name} + kwargs={"StackName": sentinel.external_name}, ) - assert response == {self.stack.name: [ - { - "LogicalResourceId": sentinel.logical_resource_id, - "PhysicalResourceId": sentinel.physical_resource_id - } - ]} + assert response == { + self.stack.name: [ + { + "LogicalResourceId": sentinel.logical_resource_id, + "PhysicalResourceId": sentinel.physical_resource_id, + } + ] + } @patch("sceptre.plan.actions.StackActions._describe") def test_describe_outputs_sends_correct_request(self, mock_describe): - mock_describe.return_value = { - "Stacks": [{ - "Outputs": sentinel.outputs - }] - } + mock_describe.return_value = {"Stacks": [{"Outputs": sentinel.outputs}]} response = self.actions.describe_outputs() mock_describe.assert_called_once_with() assert response == {self.stack.name: sentinel.outputs} @patch("sceptre.plan.actions.StackActions._describe") - def test_describe_outputs_handles_stack_with_no_outputs( - self, mock_describe - ): - mock_describe.return_value = { - "Stacks": [{}] - } + def test_describe_outputs_handles_stack_with_no_outputs(self, mock_describe): + mock_describe.return_value = {"Stacks": [{}]} response = self.actions.describe_outputs() assert response == {self.stack.name: []} @@ -540,8 +575,8 @@ def test_continue_update_rollback_sends_correct_request(self): command="continue_update_rollback", kwargs={ "StackName": sentinel.external_name, - "RoleARN": sentinel.role_arn - } + "RoleARN": sentinel.cloudformation_service_role, + }, ) def test_set_stack_policy_sends_correct_request(self): @@ -561,24 +596,22 @@ def test_set_stack_policy_sends_correct_request(self): } ] } -""" - } +""", + }, ) @patch("sceptre.plan.actions.json") def test_get_stack_policy_sends_correct_request(self, mock_Json): - mock_Json.loads.return_value = '{}' - mock_Json.dumps.return_value = '{}' + mock_Json.loads.return_value = "{}" + mock_Json.dumps.return_value = "{}" response = self.actions.get_policy() self.actions.connection_manager.call.assert_called_with( service="cloudformation", command="get_stack_policy", - kwargs={ - "StackName": sentinel.external_name - } + kwargs={"StackName": sentinel.external_name}, ) - assert response == {'prod/app/stack': '{}'} + assert response == {"prod/app/stack": "{}"} def test_create_change_set_sends_correct_request(self): self.template._body = sentinel.template @@ -590,20 +623,17 @@ def test_create_change_set_sends_correct_request(self): kwargs={ "StackName": sentinel.external_name, "TemplateBody": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], "ChangeSetName": sentinel.change_set_name, - "RoleARN": sentinel.role_arn, + "RoleARN": sentinel.cloudformation_service_role, "NotificationARNs": [sentinel.notification], - "Tags": [ - {"Key": "tag1", "Value": "val1"} - ] - } + "Tags": [{"Key": "tag1", "Value": "val1"}], + }, ) def test_create_change_set_sends_correct_request_no_notifications(self): @@ -620,20 +650,17 @@ def test_create_change_set_sends_correct_request_no_notifications(self): kwargs={ "StackName": sentinel.external_name, "Template": sentinel.template, - "Parameters": [{ - "ParameterKey": "key1", - "ParameterValue": "val1" - }], - "Capabilities": ['CAPABILITY_IAM', - 'CAPABILITY_NAMED_IAM', - 'CAPABILITY_AUTO_EXPAND'], + "Parameters": [{"ParameterKey": "key1", "ParameterValue": "val1"}], + "Capabilities": [ + "CAPABILITY_IAM", + "CAPABILITY_NAMED_IAM", + "CAPABILITY_AUTO_EXPAND", + ], "ChangeSetName": sentinel.change_set_name, - "RoleARN": sentinel.role_arn, + "RoleARN": sentinel.cloudformation_service_role, "NotificationARNs": [], - "Tags": [ - {"Key": "tag1", "Value": "val1"} - ] - } + "Tags": [{"Key": "tag1", "Value": "val1"}], + }, ) def test_delete_change_set_sends_correct_request(self): @@ -643,8 +670,8 @@ def test_delete_change_set_sends_correct_request(self): command="delete_change_set", kwargs={ "ChangeSetName": sentinel.change_set_name, - "StackName": sentinel.external_name - } + "StackName": sentinel.external_name, + }, ) def test_describe_change_set_sends_correct_request(self): @@ -654,38 +681,106 @@ def test_describe_change_set_sends_correct_request(self): command="describe_change_set", kwargs={ "ChangeSetName": sentinel.change_set_name, - "StackName": sentinel.external_name - } + "StackName": sentinel.external_name, + }, ) @patch("sceptre.plan.actions.StackActions._wait_for_completion") - def test_execute_change_set_sends_correct_request( - self, mock_wait_for_completion - ): + def test_execute_change_set_sends_correct_request(self, mock_wait_for_completion): self.actions.execute_change_set(sentinel.change_set_name) self.actions.connection_manager.call.assert_called_with( service="cloudformation", command="execute_change_set", kwargs={ "ChangeSetName": sentinel.change_set_name, - "StackName": sentinel.external_name - } + "StackName": sentinel.external_name, + }, ) - mock_wait_for_completion.assert_called_once_with() + mock_wait_for_completion.assert_called_once_with(boto_response=ANY) + + def test_execute_change_set__change_set_is_failed_for_no_changes__returns_0(self): + def fake_describe(service, command, kwargs): + assert (service, command) == ("cloudformation", "describe_change_set") + return { + "Status": "FAILED", + "StatusReason": "The submitted information didn't contain changes", + } + + self.actions.connection_manager.call.side_effect = fake_describe + result = self.actions.execute_change_set(sentinel.change_set_name) + assert result == 0 + + def test_execute_change_set__change_set_is_failed_for_no_updates__returns_0(self): + def fake_describe(service, command, kwargs): + assert (service, command) == ("cloudformation", "describe_change_set") + return { + "Status": "FAILED", + "StatusReason": "No updates are to be performed", + } + + self.actions.connection_manager.call.side_effect = fake_describe + result = self.actions.execute_change_set(sentinel.change_set_name) + assert result == 0 def test_list_change_sets_sends_correct_request(self): self.actions.list_change_sets() self.actions.connection_manager.call.assert_called_with( service="cloudformation", command="list_change_sets", - kwargs={"StackName": sentinel.external_name} + kwargs={"StackName": sentinel.external_name}, ) + @patch("sceptre.plan.actions.StackActions._list_change_sets") + def test_list_change_sets(self, mock_list_change_sets): + mock_list_change_sets_return_value = {"Summaries": []} + expected_responses = [] + + for num in ["1", "2"]: + response = [ + {"ChangeSetId": "mychangesetid{num}", "StackId": "mystackid{num}"} + ] + mock_list_change_sets_return_value["Summaries"].append(response) + expected_responses.append(response) + + mock_list_change_sets.return_value = mock_list_change_sets_return_value + + response = self.actions.list_change_sets(url=False) + assert response == {"prod/app/stack": expected_responses} + + @patch("sceptre.plan.actions.urllib.parse.urlencode") + @patch("sceptre.plan.actions.StackActions._list_change_sets") + def test_list_change_sets_url_mode(self, mock_list_change_sets, mock_urlencode): + mock_list_change_sets_return_value = {"Summaries": []} + mock_urlencode_side_effect = [] + expected_urls = [] + + for num in ["1", "2"]: + mock_list_change_sets_return_value["Summaries"].append( + {"ChangeSetId": "mychangesetid{num}", "StackId": "mystackid{num}"} + ) + urlencoded = "stackId=mystackid{num}&changeSetId=mychangesetid{num}" + mock_urlencode_side_effect.append(urlencoded) + expected_urls.append( + "https://sentinel.region.console.aws.amazon.com/cloudformation/home?" + f"region=sentinel.region#/stacks/changesets/changes?{urlencoded}" + ) + + mock_list_change_sets.return_value = mock_list_change_sets_return_value + mock_urlencode.side_effect = mock_urlencode_side_effect + + response = self.actions.list_change_sets(url=True) + assert response == {"prod/app/stack": expected_urls} + + @pytest.mark.parametrize("url_mode", [True, False]) + @patch("sceptre.plan.actions.StackActions._list_change_sets") + def test_list_change_sets_empty(self, mock_list_change_sets, url_mode): + mock_list_change_sets.return_value = {"Summaries": []} + response = self.actions.list_change_sets(url=url_mode) + assert response == {"prod/app/stack": []} + @patch("sceptre.plan.actions.StackActions.set_policy") @patch("os.path.join") - def test_lock_calls_set_stack_policy_with_policy( - self, mock_join, mock_set_policy - ): + def test_lock_calls_set_stack_policy_with_policy(self, mock_join, mock_set_policy): mock_join.return_value = "tests/fixtures/stack_policies/lock.json" self.actions.lock() mock_set_policy.assert_called_once_with( @@ -695,7 +790,7 @@ def test_lock_calls_set_stack_policy_with_policy( @patch("sceptre.plan.actions.StackActions.set_policy") @patch("os.path.join") def test_unlock_calls_set_stack_policy_with_policy( - self, mock_join, mock_set_policy + self, mock_join, mock_set_policy ): mock_join.return_value = "tests/fixtures/stack_policies/unlock.json" self.actions.unlock() @@ -704,111 +799,92 @@ def test_unlock_calls_set_stack_policy_with_policy( ) def test_format_parameters_with_sting_values(self): - parameters = { - "key1": "value1", - "key2": "value2", - "key3": "value3" - } + parameters = {"key1": "value1", "key2": "value2", "key3": "value3"} formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1"}, {"ParameterKey": "key2", "ParameterValue": "value2"}, - {"ParameterKey": "key3", "ParameterValue": "value3"} + {"ParameterKey": "key3", "ParameterValue": "value3"}, ] def test_format_parameters_with_none_values(self): - parameters = { - "key1": None, - "key2": None, - "key3": None - } + parameters = {"key1": None, "key2": None, "key3": None} formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [] def test_format_parameters_with_none_and_string_values(self): - parameters = { - "key1": "value1", - "key2": None, - "key3": "value3" - } + parameters = {"key1": "value1", "key2": None, "key3": "value3"} formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1"}, - {"ParameterKey": "key3", "ParameterValue": "value3"} + {"ParameterKey": "key3", "ParameterValue": "value3"}, ] def test_format_parameters_with_list_values(self): parameters = { "key1": ["value1", "value2", "value3"], "key2": ["value4", "value5", "value6"], - "key3": ["value7", "value8", "value9"] + "key3": ["value7", "value8", "value9"], } formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1,value2,value3"}, {"ParameterKey": "key2", "ParameterValue": "value4,value5,value6"}, - {"ParameterKey": "key3", "ParameterValue": "value7,value8,value9"} + {"ParameterKey": "key3", "ParameterValue": "value7,value8,value9"}, ] def test_format_parameters_with_none_and_list_values(self): parameters = { "key1": ["value1", "value2", "value3"], "key2": None, - "key3": ["value7", "value8", "value9"] + "key3": ["value7", "value8", "value9"], } formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1,value2,value3"}, - {"ParameterKey": "key3", "ParameterValue": "value7,value8,value9"} + {"ParameterKey": "key3", "ParameterValue": "value7,value8,value9"}, ] def test_format_parameters_with_list_and_string_values(self): parameters = { "key1": ["value1", "value2", "value3"], "key2": "value4", - "key3": ["value5", "value6", "value7"] + "key3": ["value5", "value6", "value7"], } formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1,value2,value3"}, {"ParameterKey": "key2", "ParameterValue": "value4"}, - {"ParameterKey": "key3", "ParameterValue": "value5,value6,value7"} + {"ParameterKey": "key3", "ParameterValue": "value5,value6,value7"}, ] def test_format_parameters_with_none_list_and_string_values(self): parameters = { "key1": ["value1", "value2", "value3"], "key2": "value4", - "key3": None + "key3": None, } formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1,value2,value3"}, @@ -817,9 +893,7 @@ def test_format_parameters_with_none_list_and_string_values(self): @patch("sceptre.plan.actions.StackActions._describe") def test_get_status_with_created_stack(self, mock_describe): - mock_describe.return_value = { - "Stacks": [{"StackStatus": "CREATE_COMPLETE"}] - } + mock_describe.return_value = {"Stacks": [{"StackStatus": "CREATE_COMPLETE"}]} status = self.actions.get_status() assert status == "CREATE_COMPLETE" @@ -829,33 +903,30 @@ def test_get_status_with_non_existent_stack(self, mock_describe): { "Error": { "Code": "DoesNotExistException", - "Message": "Stack does not exist" + "Message": "Stack does not exist", } }, - sentinel.operation + sentinel.operation, ) assert self.actions.get_status() == "PENDING" @patch("sceptre.plan.actions.StackActions._describe") def test_get_status_with_unknown_clinet_error(self, mock_describe): mock_describe.side_effect = ClientError( - { - "Error": { - "Code": "DoesNotExistException", - "Message": "Boom!" - } - }, - sentinel.operation + {"Error": {"Code": "DoesNotExistException", "Message": "Boom!"}}, + sentinel.operation, ) with pytest.raises(ClientError): self.actions.get_status() - def test_get_role_arn_without_role(self): - self.actions.stack.role_arn = None + def test_get_cloudformation_service_role_without_role(self): + self.actions.stack.cloudformation_service_role = None assert self.actions._get_role_arn() == {} def test_get_role_arn_with_role(self): - assert self.actions._get_role_arn() == {"RoleARN": sentinel.role_arn} + assert self.actions._get_role_arn() == { + "RoleARN": sentinel.cloudformation_service_role + } def test_protect_execution_without_protection(self): # Function should do nothing if protect == False @@ -873,22 +944,25 @@ def test_protect_execution_with_protection(self): @patch("sceptre.plan.actions.StackActions._get_status") @patch("sceptre.plan.actions.StackActions._get_simplified_status") def test_wait_for_completion_calls_log_new_events( - self, mock_get_simplified_status, mock_get_status, - mock_log_new_events + self, mock_get_simplified_status, mock_get_status, mock_log_new_events ): mock_get_simplified_status.return_value = StackStatus.COMPLETE self.actions._wait_for_completion() - mock_log_new_events.assert_called_once_with() - - @pytest.mark.parametrize("test_input,expected", [ - ("ROLLBACK_COMPLETE", StackStatus.FAILED), - ("STACK_COMPLETE", StackStatus.COMPLETE), - ("STACK_IN_PROGRESS", StackStatus.IN_PROGRESS), - ("STACK_FAILED", StackStatus.FAILED) - ]) + mock_log_new_events.assert_called_once() + assert type(mock_log_new_events.mock_calls[0].args[0]) is datetime.datetime + + @pytest.mark.parametrize( + "test_input,expected", + [ + ("ROLLBACK_COMPLETE", StackStatus.FAILED), + ("STACK_COMPLETE", StackStatus.COMPLETE), + ("STACK_IN_PROGRESS", StackStatus.IN_PROGRESS), + ("STACK_FAILED", StackStatus.FAILED), + ], + ) def test_get_simplified_status_with_known_stack_statuses( - self, test_input, expected + self, test_input, expected ): response = self.actions._get_simplified_status(test_input) assert response == expected @@ -899,10 +973,8 @@ def test_get_simplified_status_with_stack_in_unknown_state(self): @patch("sceptre.plan.actions.StackActions.describe_events") def test_log_new_events_calls_describe_events(self, mock_describe_events): - mock_describe_events.return_value = { - "StackEvents": [] - } - self.actions._log_new_events() + mock_describe_events.return_value = {"StackEvents": []} + self.actions._log_new_events(datetime.datetime.utcnow()) self.actions.describe_events.assert_called_once_with() @patch("sceptre.plan.actions.StackActions.describe_events") @@ -916,7 +988,7 @@ def test_log_new_events_prints_correct_event(self, mock_describe_events): ), "LogicalResourceId": "id-2", "ResourceType": "type-2", - "ResourceStatus": "resource-status" + "ResourceStatus": "resource-status", }, { "Timestamp": datetime.datetime( @@ -925,90 +997,470 @@ def test_log_new_events_prints_correct_event(self, mock_describe_events): "LogicalResourceId": "id-1", "ResourceType": "type-1", "ResourceStatus": "resource", - "ResourceStatusReason": "User Initiated" - } + "ResourceStatusReason": "User Initiated", + }, ] } - self.actions.most_recent_event_datetime = ( + self.actions._log_new_events( datetime.datetime(2016, 3, 15, 14, 0, 0, 0, tzinfo=tzutc()) ) - self.actions._log_new_events() @patch("sceptre.plan.actions.StackActions._get_cs_status") - def test_wait_for_cs_completion_calls_get_cs_status( - self, mock_get_cs_status - ): + def test_wait_for_cs_completion_calls_get_cs_status(self, mock_get_cs_status): mock_get_cs_status.side_effect = [ - StackChangeSetStatus.PENDING, StackChangeSetStatus.READY + StackChangeSetStatus.PENDING, + StackChangeSetStatus.READY, ] self.actions.wait_for_cs_completion(sentinel.change_set_name) mock_get_cs_status.assert_called_with(sentinel.change_set_name) @patch("sceptre.plan.actions.StackActions.describe_change_set") - def test_get_cs_status_handles_all_statuses( - self, mock_describe_change_set - ): + def test_get_cs_status_handles_all_statuses(self, mock_describe_change_set): scss = StackChangeSetStatus - return_values = { # NOQA - "Status": ('CREATE_PENDING', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE', 'DELETE_COMPLETE', 'FAILED'), # NOQA - "ExecutionStatus": { # NOQA - 'UNAVAILABLE': (scss.PENDING, scss.PENDING, scss.PENDING, scss.DEFUNCT, scss.DEFUNCT), # NOQA - 'AVAILABLE': (scss.PENDING, scss.PENDING, scss.READY, scss.DEFUNCT, scss.DEFUNCT), # NOQA - 'EXECUTE_IN_PROGRESS': (scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT), # NOQA - 'EXECUTE_COMPLETE': (scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT), # NOQA - 'EXECUTE_FAILED': (scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT), # NOQA - 'OBSOLETE': (scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT, scss.DEFUNCT), # NOQA - } # NOQA - } # NOQA - - for i, status in enumerate(return_values['Status']): - for exec_status, returns in \ - return_values['ExecutionStatus'].items(): + return_values = { # NOQA + "Status": ( + "CREATE_PENDING", + "CREATE_IN_PROGRESS", + "CREATE_COMPLETE", + "DELETE_COMPLETE", + "FAILED", + ), # NOQA + "ExecutionStatus": { # NOQA + "UNAVAILABLE": ( + scss.PENDING, + scss.PENDING, + scss.PENDING, + scss.DEFUNCT, + scss.DEFUNCT, + ), # NOQA + "AVAILABLE": ( + scss.PENDING, + scss.PENDING, + scss.READY, + scss.DEFUNCT, + scss.DEFUNCT, + ), # NOQA + "EXECUTE_IN_PROGRESS": ( + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + ), # NOQA + "EXECUTE_COMPLETE": ( + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + ), # NOQA + "EXECUTE_FAILED": ( + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + ), # NOQA + "OBSOLETE": ( + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + scss.DEFUNCT, + ), # NOQA + }, # NOQA + } # NOQA + + for i, status in enumerate(return_values["Status"]): + for exec_status, returns in return_values["ExecutionStatus"].items(): mock_describe_change_set.return_value = { "Status": status, - "ExecutionStatus": exec_status + "ExecutionStatus": exec_status, } - response = self.actions._get_cs_status( - sentinel.change_set_name - ) + response = self.actions._get_cs_status(sentinel.change_set_name) assert response == returns[i] - for status in return_values['Status']: + for status in return_values["Status"]: mock_describe_change_set.return_value = { "Status": status, - "ExecutionStatus": 'UNKOWN_STATUS' + "ExecutionStatus": "UNKOWN_STATUS", } with pytest.raises(UnknownStackChangeSetStatusError): self.actions._get_cs_status(sentinel.change_set_name) - for exec_status in return_values['ExecutionStatus'].keys(): + for exec_status in return_values["ExecutionStatus"].keys(): mock_describe_change_set.return_value = { - "Status": 'UNKOWN_STATUS', - "ExecutionStatus": exec_status + "Status": "UNKOWN_STATUS", + "ExecutionStatus": exec_status, } with pytest.raises(UnknownStackChangeSetStatusError): self.actions._get_cs_status(sentinel.change_set_name) mock_describe_change_set.return_value = { - "Status": 'UNKOWN_STATUS', - "ExecutionStatus": 'UNKOWN_STATUS', + "Status": "UNKOWN_STATUS", + "ExecutionStatus": "UNKOWN_STATUS", } with pytest.raises(UnknownStackChangeSetStatusError): self.actions._get_cs_status(sentinel.change_set_name) @patch("sceptre.plan.actions.StackActions.describe_change_set") - def test_get_cs_status_raises_unexpected_exceptions( - self, mock_describe_change_set - ): + def test_get_cs_status_raises_unexpected_exceptions(self, mock_describe_change_set): mock_describe_change_set.side_effect = ClientError( { "Error": { "Code": "ChangeSetNotFound", - "Message": "ChangeSet [*] does not exist" + "Message": "ChangeSet [*] does not exist", } }, - sentinel.operation + sentinel.operation, ) with pytest.raises(ClientError): self.actions._get_cs_status(sentinel.change_set_name) + + def test_fetch_remote_template__cloudformation_returns_validation_error__returns_none( + self, + ): + self.actions.connection_manager.call.side_effect = ClientError( + { + "Error": { + "Code": "ValidationError", + "Message": "An error occurred (ValidationError) " + "when calling the GetTemplate operation: " + "Stack with id foo does not exist", + } + }, + sentinel.operation, + ) + + result = self.actions.fetch_remote_template() + assert result is None + + def test_fetch_remote_template__calls_cloudformation_get_template(self): + self.actions.connection_manager.call.return_value = {"TemplateBody": ""} + self.actions.fetch_remote_template() + + self.actions.connection_manager.call.assert_called_with( + service="cloudformation", + command="get_template", + kwargs={"StackName": self.stack.external_name, "TemplateStage": "Original"}, + ) + + def test_fetch_remote_template__dict_template__returns_json(self): + template_body = {"AWSTemplateFormatVersion": "2010-09-09", "Resources": {}} + self.actions.connection_manager.call.return_value = { + "TemplateBody": template_body + } + expected = json.dumps(template_body, indent=4) + + result = self.actions.fetch_remote_template() + assert result == expected + + def test_fetch_remote_template__cloudformation_returns_string_template__returns_that_string( + self, + ): + template_body = "This is my template" + self.actions.connection_manager.call.return_value = { + "TemplateBody": template_body + } + result = self.actions.fetch_remote_template() + assert result == template_body + + def test_fetch_remote_template_summary__calls_cloudformation_get_template_summary( + self, + ): + self.actions.fetch_remote_template_summary() + + self.actions.connection_manager.call.assert_called_with( + service="cloudformation", + command="get_template_summary", + kwargs={ + "StackName": self.stack.external_name, + }, + ) + + def test_fetch_remote_template_summary__returns_response_from_cloudformation(self): + def get_template_summary(service, command, kwargs): + assert (service, command) == ("cloudformation", "get_template_summary") + return {"template": "summary"} + + self.actions.connection_manager.call.side_effect = get_template_summary + result = self.actions.fetch_remote_template_summary() + assert result == {"template": "summary"} + + def test_fetch_local_template_summary__calls_cloudformation_get_template_summary( + self, + ): + self.actions.fetch_local_template_summary() + + self.actions.connection_manager.call.assert_called_with( + service="cloudformation", + command="get_template_summary", + kwargs={ + "TemplateBody": self.stack.template.body, + }, + ) + + def test_fetch_local_template_summary__returns_response_from_cloudformation(self): + def get_template_summary(service, command, kwargs): + assert (service, command) == ("cloudformation", "get_template_summary") + return {"template": "summary"} + + self.actions.connection_manager.call.side_effect = get_template_summary + result = self.actions.fetch_local_template_summary() + assert result == {"template": "summary"} + + def test_fetch_local_template_summary__cloudformation_returns_validation_error_invalid_stack__raises_it( + self, + ): + self.actions.connection_manager.call.side_effect = ClientError( + { + "Error": { + "Code": "ValidationError", + "Message": "Template format error: Resource name {Invalid::Resource} is " + "non alphanumeric.'", + } + }, + sentinel.operation, + ) + with pytest.raises(ClientError): + self.actions.fetch_local_template_summary() + + def test_fetch_remote_template_summary__cloudformation_returns_validation_error_for_no_stack__returns_none( + self, + ): + self.actions.connection_manager.call.side_effect = ClientError( + { + "Error": { + "Code": "ValidationError", + "Message": "An error occurred (ValidationError) " + "when calling the GetTemplate operation: " + "Stack with id foo does not exist", + } + }, + sentinel.operation, + ) + result = self.actions.fetch_remote_template_summary() + assert result is None + + def test_diff__invokes_diff_method_on_injected_differ_with_self(self): + differ = Mock() + self.actions.diff(differ) + differ.diff.assert_called_with(self.actions) + + def test_diff__returns_result_of_injected_differs_diff_method(self): + differ = Mock() + result = self.actions.diff(differ) + assert result == differ.diff.return_value + + @patch("sceptre.plan.actions.StackActions._describe_stack_drift_detection_status") + @patch("sceptre.plan.actions.StackActions._detect_stack_drift") + @patch("time.sleep") + def test_drift_detect( + self, + mock_sleep, + mock_detect_stack_drift, + mock_describe_stack_drift_detection_status, + ): + mock_sleep.return_value = None + + mock_detect_stack_drift.return_value = { + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62" + } + + first_response = { + "StackId": "fake-stack-id", + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62", + "DetectionStatus": "DETECTION_IN_PROGRESS", + "StackDriftStatus": "NOT_CHECKED", + "DetectionStatusReason": "User Initiated", + } + + final_response = { + "StackId": "fake-stack-id", + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62", + "StackDriftStatus": "IN_SYNC", + "DetectionStatus": "DETECTION_COMPLETE", + "DriftedStackResourceCount": 0, + } + + mock_describe_stack_drift_detection_status.side_effect = [ + first_response, + final_response, + ] + + response = self.actions.drift_detect() + assert response == final_response + + @pytest.mark.parametrize( + "detection_status", ["DETECTION_COMPLETE", "DETECTION_FAILED"] + ) + @patch("sceptre.plan.actions.StackActions._describe_stack_resource_drifts") + @patch("sceptre.plan.actions.StackActions._describe_stack_drift_detection_status") + @patch("sceptre.plan.actions.StackActions._detect_stack_drift") + @patch("time.sleep") + def test_drift_show( + self, + mock_sleep, + mock_detect_stack_drift, + mock_describe_stack_drift_detection_status, + mock_describe_stack_resource_drifts, + detection_status, + ): + mock_sleep.return_value = None + + mock_detect_stack_drift.return_value = { + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62" + } + mock_describe_stack_drift_detection_status.side_effect = [ + { + "StackId": "fake-stack-id", + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62", + "DetectionStatus": "DETECTION_IN_PROGRESS", + "StackDriftStatus": "FOO", + "DetectionStatusReason": "User Initiated", + }, + { + "StackId": "fake-stack-id", + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62", + "StackDriftStatus": "FOO", + "DetectionStatus": detection_status, + "DriftedStackResourceCount": 0, + }, + ] + + expected_drifts = { + "StackResourceDrifts": [ + { + "StackId": "fake-stack-id", + "LogicalResourceId": "VPC", + "PhysicalResourceId": "vpc-028c655dea7c65227", + "ResourceType": "AWS::EC2::VPC", + "ExpectedProperties": '{"foo":"bar"}', + "ActualProperties": '{"foo":"bar"}', + "PropertyDifferences": [], + "StackResourceDriftStatus": detection_status, + } + ] + } + + mock_describe_stack_resource_drifts.return_value = expected_drifts + expected_response = (detection_status, expected_drifts) + + response = self.actions.drift_show(drifted=False) + + assert response == expected_response + + @patch("sceptre.plan.actions.StackActions._describe_stack_resource_drifts") + @patch("sceptre.plan.actions.StackActions._describe_stack_drift_detection_status") + @patch("sceptre.plan.actions.StackActions._detect_stack_drift") + @patch("time.sleep") + def test_drift_show_drift_only( + self, + mock_sleep, + mock_detect_stack_drift, + mock_describe_stack_drift_detection_status, + mock_describe_stack_resource_drifts, + ): + mock_sleep.return_value = None + + mock_detect_stack_drift.return_value = { + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62" + } + mock_describe_stack_drift_detection_status.return_value = { + "StackId": "fake-stack-id", + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62", + "StackDriftStatus": "DRIFTED", + "DetectionStatus": "DETECTION_COMPLETE", + "DriftedStackResourceCount": 0, + } + + input_drifts = { + "StackResourceDrifts": [ + { + "LogicalResourceId": "ServerLoadBalancer", + "PhysicalResourceId": "bi-tablea-ServerLo-1E133TWLWYLON", + "ResourceType": "AWS::ElasticLoadBalancing::LoadBalancer", + "StackId": "fake-stack-id", + "StackResourceDriftStatus": "IN_SYNC", + }, + { + "LogicalResourceId": "TableauServer", + "PhysicalResourceId": "i-08c16bc1c5e2cd185", + "ResourceType": "AWS::EC2::Instance", + "StackId": "fake-stack-id", + "StackResourceDriftStatus": "DELETED", + }, + ] + } + mock_describe_stack_resource_drifts.return_value = input_drifts + + expected_response = ( + "DETECTION_COMPLETE", + {"StackResourceDrifts": [input_drifts["StackResourceDrifts"][1]]}, + ) + + response = self.actions.drift_show(drifted=True) + + assert response == expected_response + + @patch("sceptre.plan.actions.StackActions._get_status") + def test_drift_show_with_stack_that_does_not_exist(self, mock_get_status): + mock_get_status.side_effect = StackDoesNotExistError() + response = self.actions.drift_show(drifted=False) + assert response == ( + "STACK_DOES_NOT_EXIST", + {"StackResourceDriftStatus": "STACK_DOES_NOT_EXIST"}, + ) + + @patch("sceptre.plan.actions.StackActions._describe_stack_resource_drifts") + @patch("sceptre.plan.actions.StackActions._describe_stack_drift_detection_status") + @patch("sceptre.plan.actions.StackActions._detect_stack_drift") + @patch("time.sleep") + def test_drift_show_times_out( + self, + mock_sleep, + mock_detect_stack_drift, + mock_describe_stack_drift_detection_status, + mock_describe_stack_resource_drifts, + ): + mock_sleep.return_value = None + + mock_detect_stack_drift.return_value = { + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62" + } + + response = { + "StackId": "fake-stack-id", + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62", + "DetectionStatus": "DETECTION_IN_PROGRESS", + "StackDriftStatus": "FOO", + "DetectionStatusReason": "User Initiated", + } + + side_effect = [] + for _ in range(0, 60): + side_effect.append(response) + mock_describe_stack_drift_detection_status.side_effect = side_effect + + expected_drifts = { + "StackResourceDrifts": [ + { + "StackId": "fake-stack-id", + "LogicalResourceId": "VPC", + "PhysicalResourceId": "vpc-028c655dea7c65227", + "ResourceType": "AWS::EC2::VPC", + "ExpectedProperties": '{"foo":"bar"}', + "ActualProperties": '{"foo":"bar"}', + "PropertyDifferences": [], + "StackResourceDriftStatus": "DETECTION_IN_PROGRESS", + } + ] + } + + mock_describe_stack_resource_drifts.return_value = expected_drifts + expected_response = ("TIMED_OUT", {"StackResourceDriftStatus": "TIMED_OUT"}) + + response = self.actions.drift_show(drifted=False) + + assert response == expected_response diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 9a399c4a8..000000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,737 +0,0 @@ -import logging -import yaml -import datetime -import os -import errno -import json - -from click.testing import CliRunner -from mock import MagicMock, patch, sentinel -import pytest -import click - -from sceptre.cli import cli -from sceptre.config.reader import ConfigReader -from sceptre.stack import Stack -from sceptre.plan.actions import StackActions -from sceptre.stack_status import StackStatus -from sceptre.cli.helpers import setup_logging, write, ColouredFormatter -from sceptre.cli.helpers import CustomJsonEncoder, catch_exceptions -from botocore.exceptions import ClientError -from sceptre.exceptions import SceptreException - - -class TestCli(object): - - def setup_method(self, test_method): - self.patcher_ConfigReader = patch("sceptre.plan.plan.ConfigReader") - self.patcher_StackActions = patch("sceptre.plan.executor.StackActions") - - self.mock_ConfigReader = self.patcher_ConfigReader.start() - self.mock_StackActions = self.patcher_StackActions.start() - - self.mock_config_reader = MagicMock(spec=ConfigReader) - self.mock_stack_actions = MagicMock(spec=StackActions) - - self.mock_stack = MagicMock(spec=Stack) - - self.mock_stack.name = 'mock-stack' - self.mock_stack.region = None - self.mock_stack.profile = None - self.mock_stack.external_name = None - self.mock_stack.dependencies = [] - - self.mock_config_reader.construct_stacks.return_value = \ - set([self.mock_stack]), set([self.mock_stack]) - - self.mock_stack_actions.stack = self.mock_stack - - self.mock_ConfigReader.return_value = self.mock_config_reader - self.mock_StackActions.return_value = self.mock_stack_actions - - self.runner = CliRunner() - - def teardown_method(self, test_method): - self.patcher_ConfigReader.stop() - self.patcher_StackActions.stop() - - @patch("sys.exit") - def test_catch_excecptions(self, mock_exit): - @catch_exceptions - def raises_exception(): - raise SceptreException() - - raises_exception() - mock_exit.assert_called_once_with(1) - - @pytest.mark.parametrize("command,files,output", [ - # one --var option - ( - ["--var", "a=1", "noop"], - {}, - {"a": "1"} - ), - # multiple --var options - ( - ["--var", "a=1", "--var", "b=2", "noop"], - {}, - {"a": "1", "b": "2"} - ), - # multiple --var options same key - ( - ["--var", "a=1", "--var", "a=2", "noop"], - {}, - {"a": "2"} - ), - ( - ["--var-file", "foo.yaml", "--var", "key3.subkey1.id=id2", "noop"], - { - "foo.yaml": { - "key1": "val1", - "key2": "val2", - "key3": { - "subkey1": { - "id": "id1" - } - } - } - }, - { - "key1": "val1", - "key2": "val2", - "key3": {"subkey1": {"id": "id2"}} - } - ), - # one --var-file option - ( - ["--var-file", "foo.yaml", "noop"], - { - "foo.yaml": {"key1": "val1", "key2": "val2"} - }, - {"key1": "val1", "key2": "val2"} - ), - # multiple --var-file option - ( - ["--var-file", "foo.yaml", "--var-file", "bar.yaml", "noop"], - { - "foo.yaml": {"key1": "parent_value1", "key2": "parent_value2"}, - "bar.yaml": {"key2": "child_value2", "key3": "child_value3"} - }, - { - "key1": "parent_value1", - "key2": "child_value2", - "key3": "child_value3" - } - ), - # mix of --var and --var-file - ( - ["--var-file", "foo.yaml", "--var", "key2=var2", "noop"], - { - "foo.yaml": {"key1": "file1", "key2": "file2"} - }, - {"key1": "file1", "key2": "var2"} - ), - ]) - def test_user_variables(self, command, files, output): - @cli.command() - @click.pass_context - def noop(ctx): - click.echo(yaml.safe_dump(ctx.obj.get("user_variables"))) - - with self.runner.isolated_filesystem(): - for name, content in files.items(): - with open(name, "w") as fh: - yaml.safe_dump(content, fh) - - result = self.runner.invoke(cli, command) - - user_variables = yaml.safe_load(result.output) - assert result.exit_code == 0 - assert user_variables == output - - def test_validate_template_with_valid_template(self): - self.mock_stack_actions.validate.return_value = { - "Parameters": "Example", - "ResponseMetadata": { - "HTTPStatusCode": 200 - } - } - - result_json = json.dumps({'Parameters': 'Example'}, indent=4) - result = self.runner.invoke(cli, ["validate", "dev/vpc.yaml"]) - self.mock_stack_actions.validate.assert_called_with() - assert result.output == "Template mock-stack is valid. Template details:\n\n{}\n".format( - result_json) - - def test_validate_template_with_invalid_template(self): - client_error = ClientError( - { - "Errors": - { - "Message": "Unrecognized resource types", - "Code": "ValidationError", - } - }, - "ValidateTemplate" - ) - self.mock_stack_actions.validate.side_effect = client_error - - expected_result = str(client_error) + "\n" - result = self.runner.invoke(cli, ["validate", "dev/vpc.yaml"]) - assert expected_result in result.output.replace("\"", "") - - def test_estimate_template_cost_with_browser(self): - self.mock_stack_actions.estimate_cost.return_value = { - "Url": "https://sceptre.cloudreach.com", - "ResponseMetadata": { - "HTTPStatusCode": 200 - } - } - - args = ["estimate-cost", "dev/vpc.yaml"] - result = self.runner.invoke(cli, args) - - self.mock_stack_actions.estimate_cost.assert_called_with() - - assert result.output == \ - '{0}{1}'.format("View the estimated cost for mock-stack at:\n", - "https://sceptre.cloudreach.com\n\n") - - def test_estimate_template_cost_with_no_browser(self): - client_error = ClientError( - { - "Errors": - { - "Message": "No Browser", - "Code": "Error", - } - }, - "Webbrowser" - ) - self.mock_stack_actions.estimate_cost.side_effect = client_error - expected_result = "{}\n".format(client_error) - result = self.runner.invoke( - cli, - ["estimate-cost", "dev/vpc.yaml"] - ) - assert expected_result in result.output.replace("\"", "") - - def test_lock_stack(self): - self.runner.invoke( - cli, ["set-policy", "dev/vpc.yaml", "-b", "deny-all"] - ) - self.mock_config_reader.construct_stacks.assert_called_with() - self.mock_stack_actions.lock.assert_called_with() - - def test_unlock_stack(self): - self.runner.invoke( - cli, ["set-policy", "dev/vpc.yaml", "-b", "allow-all"] - ) - self.mock_config_reader.construct_stacks.assert_called_with() - self.mock_stack_actions.unlock.assert_called_with() - - def test_set_policy_with_file_flag(self): - policy_file = "tests/fixtures/stack_policies/lock.json" - result = self.runner.invoke(cli, [ - "set-policy", "dev/vpc.yaml", policy_file - ]) - assert result.exit_code == 0 - - def test_describe_policy_with_existing_policy(self): - self.mock_stack_actions.get_policy.return_value = { - "dev/vpc": {"Statement": ["Body"]} - } - - result = self.runner.invoke( - cli, ["describe", "policy", "dev/vpc.yaml"] - ) - assert result.exit_code == 0 - assert result.output == "{}\n".format(json.dumps( - {'dev/vpc': {'Statement': ['Body']}}, indent=4)) - - def test_list_group_resources(self): - response = { - "stack-name-1": { - "StackResources": [ - { - "LogicalResourceId": "logical-resource-id", - "PhysicalResourceId": "physical-resource-id" - } - ] - }, - "stack-name-2": { - "StackResources": [ - { - "LogicalResourceId": "logical-resource-id", - "PhysicalResourceId": "physical-resource-id" - } - ] - } - } - self.mock_stack_actions.describe_resources.return_value = response - result = self.runner.invoke(cli, ["list", "resources", "dev"]) - - assert yaml.safe_load(result.output) == [response] - assert result.exit_code == 0 - - def test_list_stack_resources(self): - response = { - "StackResources": [ - { - "LogicalResourceId": "logical-resource-id", - "PhysicalResourceId": "physical-resource-id" - } - ] - } - self.mock_stack_actions.describe_resources.return_value = response - result = self.runner.invoke(cli, ["list", "resources", "dev/vpc.yaml"]) - assert yaml.safe_load(result.output) == [response] - assert result.exit_code == 0 - - @pytest.mark.parametrize( - "command,success,yes_flag,exit_code", [ - ("create", True, True, 0), - ("create", False, True, 1), - ("create", True, False, 0), - ("create", False, False, 1), - ("delete", True, True, 0), - ("delete", False, True, 1), - ("delete", True, False, 0), - ("delete", False, False, 1), - ("update", True, True, 0), - ("update", False, True, 1), - ("update", True, False, 0), - ("update", False, False, 1), - ("launch", True, True, 0), - ("launch", False, True, 1), - ("launch", True, False, 0), - ("launch", False, False, 1) - ] - ) - def test_stack_commands(self, command, success, yes_flag, exit_code): - run_command = getattr(self.mock_stack_actions, command) - run_command.return_value = \ - StackStatus.COMPLETE if success else StackStatus.FAILED - - kwargs = {"args": [command, "dev/vpc.yaml"]} - if yes_flag: - kwargs["args"].append("-y") - else: - kwargs["input"] = "y\n" - - result = self.runner.invoke(cli, **kwargs) - - run_command.assert_called_with() - assert result.exit_code == exit_code - - @pytest.mark.parametrize( - "command, ignore_dependencies", [ - ("create", True), - ("create", False), - ("delete", True), - ("delete", False), - ] - ) - def test_ignore_dependencies_commands(self, command, ignore_dependencies): - args = [command, "dev/vpc.yaml", "cs-1", "-y"] - if ignore_dependencies: - args.insert(0, "--ignore-dependencies") - result = self.runner.invoke(cli, args) - assert result.exit_code == 0 - - @pytest.mark.parametrize( - "command,yes_flag", [ - ("create", True), - ("create", False), - ("delete", True), - ("delete", False), - ("execute", True), - ("execute", False) - ] - ) - def test_change_set_commands(self, command, yes_flag): - stack_command = command + "_change_set" - - kwargs = {"args": [command, "dev/vpc.yaml", "cs1"]} - if yes_flag: - kwargs["args"].append("-y") - else: - kwargs["input"] = "y\n" - - result = self.runner.invoke(cli, **kwargs) - - getattr(self.mock_stack_actions, - stack_command).assert_called_with("cs1") - assert result.exit_code == 0 - - @pytest.mark.parametrize( - "verbose_flag,", [ - (False), - (True) - ] - ) - def test_describe_change_set(self, verbose_flag): - response = { - "VerboseProperty": "VerboseProperty", - "ChangeSetName": "ChangeSetName", - "CreationTime": "CreationTime", - "ExecutionStatus": "ExecutionStatus", - "StackName": "StackName", - "Status": "Status", - "StatusReason": "StatusReason", - "Changes": [ - { - "ResourceChange": { - "Action": "Action", - "LogicalResourceId": "LogicalResourceId", - "PhysicalResourceId": "PhysicalResourceId", - "Replacement": "Replacement", - "ResourceType": "ResourceType", - "Scope": "Scope", - "VerboseProperty": "VerboseProperty" - } - } - ] - } - args = ["describe", "change-set", "region/vpc.yaml", "cs1"] - if verbose_flag: - args.append("-v") - - self.mock_stack_actions.describe_change_set.return_value = response - result = self.runner.invoke(cli, args) - if not verbose_flag: - del response["VerboseProperty"] - del response["Changes"][0]["ResourceChange"]["VerboseProperty"] - assert yaml.safe_load(result.output) == response - assert result.exit_code == 0 - - def test_list_change_sets_with_200(self): - self.mock_stack_actions.list_change_sets.return_value = { - "ChangeSets": "Test" - } - result = self.runner.invoke( - cli, ["list", "change-sets", "dev/vpc.yaml"] - ) - assert result.exit_code == 0 - assert yaml.safe_load(result.output) == {"ChangeSets": "Test"} - - def test_list_change_sets_without_200(self): - response = { - "ChangeSets": "Test" - } - self.mock_stack_actions.list_change_sets.return_value = response - - result = self.runner.invoke( - cli, ["list", "change-sets", "dev/vpc.yaml"] - ) - assert result.exit_code == 0 - assert yaml.safe_load(result.output) == response - - def test_list_outputs(self): - outputs = {"OutputKey": "Key", "OutputValue": "Value"} - self.mock_stack_actions.describe_outputs.return_value = outputs - result = self.runner.invoke( - cli, ["list", "outputs", "dev/vpc.yaml"] - ) - assert result.exit_code == 0 - assert json.loads(result.output) == [outputs] - - def test_list_outputs_with_export(self): - outputs = {'stack': [{'OutputKey': 'Key', 'OutputValue': 'Value'}]} - self.mock_stack_actions.describe_outputs.return_value = outputs - result = self.runner.invoke( - cli, ["list", "outputs", "dev/vpc.yaml", "-e", "envvar"] - ) - assert result.exit_code == 0 - assert result.output == "export SCEPTRE_Key='Value'\n" - - def test_status_with_group(self): - self.mock_stack_actions.get_status.return_value = { - "stack": "status" - } - - result = self.runner.invoke(cli, ["status", "dev"]) - assert result.exit_code == 0 - assert result.output == '{\n "mock-stack": {\n \"stack\": \"status\"\n }\n}\n' - - def test_status_with_stack(self): - self.mock_stack_actions.get_status.return_value = "status" - result = self.runner.invoke(cli, ["status", "dev/vpc.yaml"]) - assert result.exit_code == 0 - assert result.output == '{\n "mock-stack": "status"\n}\n' - - def test_new_project_non_existant(self): - with self.runner.isolated_filesystem(): - project_path = os.path.abspath('./example') - config_dir = os.path.join(project_path, "config") - template_dir = os.path.join(project_path, "templates") - region = "test-region" - os.environ["AWS_DEFAULT_REGION"] = region - defaults = { - "project_code": "example", - "region": region - } - - result = self.runner.invoke(cli, ["new", "project", "example"]) - assert not result.exception - assert os.path.isdir(config_dir) - assert os.path.isdir(template_dir) - - with open(os.path.join(config_dir, "config.yaml")) as config_file: - config = yaml.safe_load(config_file) - - assert config == defaults - - def test_new_project_already_exist(self): - with self.runner.isolated_filesystem(): - project_path = os.path.abspath('./example') - config_dir = os.path.join(project_path, "config") - template_dir = os.path.join(project_path, "templates") - existing_config = {"Test": "Test"} - - os.mkdir(project_path) - os.mkdir(config_dir) - os.mkdir(template_dir) - - config_filepath = os.path.join(config_dir, "config.yaml") - with open(config_filepath, 'w') as config_file: - yaml.dump(existing_config, config_file) - - result = self.runner.invoke(cli, ["new", "project", "example"]) - assert result.exit_code == 1 - assert result.output == '"Folder \\"example\\" already exists."\n' - assert os.path.isdir(config_dir) - assert os.path.isdir(template_dir) - - with open(os.path.join(config_dir, "config.yaml")) as config_file: - config = yaml.safe_load(config_file) - assert existing_config == config - - def test_new_project_another_exception(self): - with self.runner.isolated_filesystem(): - patcher_mkdir = patch("sceptre.cli.new.os.mkdir") - mock_mkdir = patcher_mkdir.start() - mock_mkdir.side_effect = OSError(errno.EINVAL) - result = self.runner.invoke(cli, ["new", "project", "example"]) - mock_mkdir = patcher_mkdir.stop() - assert str(result.exception) == str(OSError(errno.EINVAL)) - - @pytest.mark.parametrize( - "stack_group,config_structure,stdin,result", [ - ( - "A", - {"": {}}, - 'y\nA\nA\n', {"project_code": "A", "region": "A"} - ), - ( - "A", - {"": {"project_code": "top", "region": "top"}}, - 'y\n\n\n', {} - ), - ( - "A", - {"": {"project_code": "top", "region": "top"}}, - 'y\nA\nA\n', {"project_code": "A", "region": "A"} - ), - ( - "A/A", - { - "": {"project_code": "top", "region": "top"}, - "A": {"project_code": "A", "region": "A"}, - }, - 'y\nA/A\nA/A\n', {"project_code": "A/A", "region": "A/A"} - ), - ( - "A/A", - { - "": {"project_code": "top", "region": "top"}, - "A": {"project_code": "A", "region": "A"}, - }, - 'y\nA\nA\n', {} - ) - ] - ) - def test_create_new_stack_group_folder( - self, stack_group, config_structure, stdin, result - ): - with self.runner.isolated_filesystem(): - project_path = os.path.abspath('./example') - config_dir = os.path.join(project_path, "config") - os.makedirs(config_dir) - - stack_group_dir = os.path.join(project_path, "config", stack_group) - for stack_group_path, config in config_structure.items(): - path = os.path.join(config_dir, stack_group_path) - try: - os.makedirs(path) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - filepath = os.path.join(path, "config.yaml") - with open(filepath, 'w') as config_file: - yaml.safe_dump( - config, stream=config_file, default_flow_style=False - ) - - os.chdir(project_path) - - cmd_result = self.runner.invoke( - cli, ["new", "group", stack_group], - input=stdin - ) - - if result: - with open(os.path.join(stack_group_dir, "config.yaml"))\ - as config_file: - config = yaml.safe_load(config_file) - assert config == result - else: - assert cmd_result.output.endswith( - "No config.yaml file needed - covered by parent config.\n" - ) - - def test_new_stack_group_folder_with_existing_folder(self): - with self.runner.isolated_filesystem(): - project_path = os.path.abspath('./example') - config_dir = os.path.join(project_path, "config") - stack_group_dir = os.path.join(config_dir, "A") - - os.makedirs(stack_group_dir) - os.chdir(project_path) - - cmd_result = self.runner.invoke( - cli, ["new", "group", "A"], input="y\n\n\n" - ) - - assert cmd_result.output.startswith( - "StackGroup path exists. " - "Do you want initialise config.yaml?" - ) - with open(os.path.join( - stack_group_dir, "config.yaml")) as config_file: - config = yaml.safe_load(config_file) - assert config == {"project_code": "", "region": ""} - - def test_new_stack_group_folder_with_another_exception(self): - with self.runner.isolated_filesystem(): - project_path = os.path.abspath('./example') - config_dir = os.path.join(project_path, "config") - stack_group_dir = os.path.join(config_dir, "A") - - os.makedirs(stack_group_dir) - os.chdir(project_path) - patcher_mkdir = patch("sceptre.cli.new.os.mkdir") - mock_mkdir = patcher_mkdir.start() - mock_mkdir.side_effect = OSError(errno.EINVAL) - result = self.runner.invoke(cli, ["new", "group", "A"]) - mock_mkdir = patcher_mkdir.stop() - assert str(result.exception) == str(OSError(errno.EINVAL)) - - @pytest.mark.parametrize( - "cli_module,command,output_format,no_colour", [ - ( - 'describe', - ['describe', 'change-set', 'somepath', 'cs1'], - 'yaml', - True - ), - ( - 'describe', - ['describe', 'change-set', 'somepath', 'cs1'], - 'json', - False - ), - ( - 'describe', - ['describe', 'policy', 'somepolicy'], - 'yaml', - True - ), - ( - 'describe', - ['describe', 'policy', 'somepolicy'], - 'json', - False - ) - ] - ) - def test_write_output_format_flags( - self, cli_module, command, output_format, no_colour - ): - no_colour_flag = ['--no-colour'] if no_colour else [] - output_format_flag = ['--output', output_format] - args = output_format_flag + no_colour_flag + command - - with patch("sceptre.cli." + cli_module + ".write") as mock_write: - self.runner.invoke(cli, args) - mock_write.assert_called() - for call in mock_write.call_args_list: - args, _ = call - assert args[1] == output_format - assert args[2] == no_colour - - def test_setup_logging_with_debug(self): - logger = setup_logging(True, False) - assert logger.getEffectiveLevel() == logging.DEBUG - assert logging.getLogger("botocore").getEffectiveLevel() == \ - logging.INFO - - # Silence logging for the rest of the tests - logger.setLevel(logging.CRITICAL) - - def test_setup_logging_without_debug(self): - logger = setup_logging(False, False) - assert logger.getEffectiveLevel() == logging.INFO - assert logging.getLogger("botocore").getEffectiveLevel() == \ - logging.CRITICAL - - # Silence logging for the rest of the tests - logger.setLevel(logging.CRITICAL) - - @patch("sceptre.cli.click.echo") - @pytest.mark.parametrize( - "output_format,no_colour,expected_output", [ - ("json", True, '{\n "stack": "CREATE_COMPLETE"\n}'), - ("json", False, '{\n "stack": "\x1b[32mCREATE_COMPLETE\x1b[0m\"\n}'), - ("yaml", True, {'stack': 'CREATE_COMPLETE'}), - ("yaml", False, '{\'stack\': \'\x1b[32mCREATE_COMPLETE\x1b[0m\'}') - ] - ) - def test_write_formats( - self, mock_echo, output_format, no_colour, expected_output - ): - write({"stack": "CREATE_COMPLETE"}, output_format, no_colour) - mock_echo.assert_called_once_with(expected_output) - - @patch("sceptre.cli.click.echo") - def test_write_status_with_colour(self, mock_echo): - write("stack: CREATE_COMPLETE", no_colour=False) - mock_echo.assert_called_once_with( - '{\n "stack": "\x1b[32mCREATE_COMPLETE\x1b[0m"\n}' - ) - - @patch("sceptre.cli.click.echo") - def test_write_status_without_colour(self, mock_echo): - write("stack: CREATE_COMPLETE", no_colour=True) - mock_echo.assert_called_once_with('{\n "stack": "CREATE_COMPLETE"\n}') - - @patch("sceptre.cli.helpers.StackStatusColourer.colour") - @patch("sceptre.cli.helpers.logging.Formatter.format") - def test_ColouredFormatter_format_with_string( - self, mock_format, mock_colour - ): - mock_format.return_value = sentinel.response - mock_colour.return_value = sentinel.coloured_response - coloured_formatter = ColouredFormatter() - response = coloured_formatter.format("string") - mock_format.assert_called_once_with("string") - mock_colour.assert_called_once_with(sentinel.response) - assert response == sentinel.coloured_response - - def test_CustomJsonEncoder_with_non_json_serialisable_object(self): - encoder = CustomJsonEncoder() - response = encoder.encode(datetime.datetime(2016, 5, 3)) - assert response == '"2016-05-03 00:00:00"' diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_cli/test_cli_commands.py b/tests/test_cli/test_cli_commands.py new file mode 100644 index 000000000..c3b4fd518 --- /dev/null +++ b/tests/test_cli/test_cli_commands.py @@ -0,0 +1,1048 @@ +import datetime +import errno +import json +import logging +import os +from copy import deepcopy + +import click +import pytest +import yaml + +from botocore.exceptions import ClientError +from click.testing import CliRunner +from deepdiff import DeepDiff +from unittest.mock import MagicMock, patch, sentinel + +from sceptre.cli import cli +from sceptre.cli.helpers import ( + CustomJsonEncoder, + catch_exceptions, + setup_logging, + write, + ColouredFormatter, + deserialize_json_properties, +) + +from sceptre.config.reader import ConfigReader + +from sceptre.diffing.stack_differ import ( + DeepDiffStackDiffer, + DifflibStackDiffer, + StackDiff, +) + +from sceptre.exceptions import SceptreException +from sceptre.plan.actions import StackActions +from sceptre.stack import Stack +from sceptre.stack_status import StackStatus + + +class TestCli: + def setup_method(self, test_method): + self.patcher_ConfigReader = patch("sceptre.plan.plan.ConfigReader") + self.patcher_StackActions = patch("sceptre.plan.executor.StackActions") + + self.mock_ConfigReader = self.patcher_ConfigReader.start() + self.mock_StackActions = self.patcher_StackActions.start() + + self.mock_config_reader = MagicMock(spec=ConfigReader) + self.mock_stack_actions = MagicMock(spec=StackActions) + + self.mock_stack = MagicMock( + spec=Stack, + region=None, + profile=None, + external_name="mock-stack-external", + dependencies=[], + ignore=False, + obsolete=False, + ) + + self.mock_stack.name = "mock-stack" + + self.mock_config_reader.construct_stacks.return_value = set( + [self.mock_stack] + ), set([self.mock_stack]) + + self.mock_stack_actions.stack = self.mock_stack + + self.mock_ConfigReader.return_value = self.mock_config_reader + self.mock_StackActions.return_value = self.mock_stack_actions + + self.runner = CliRunner() + + def teardown_method(self, test_method): + self.patcher_ConfigReader.stop() + self.patcher_StackActions.stop() + + @patch("sys.exit") + def test_catch_exceptions(self, mock_exit): + @catch_exceptions + def raises_exception(): + raise SceptreException() + + raises_exception() + mock_exit.assert_called_once_with(1) + + def test_catch_exceptions_debug_mode(self): + @catch_exceptions + def raises_exception(): + raise SceptreException() + + logger = logging.getLogger("sceptre") + logger.setLevel(logging.DEBUG) + + with pytest.raises(SceptreException): + raises_exception() + + @pytest.mark.parametrize( + "command,files,output", + [ + # one --var option + (["--var", "a=1", "noop"], {}, {"a": "1"}), + # multiple --var options + (["--var", "a=1", "--var", "b=2", "noop"], {}, {"a": "1", "b": "2"}), + # multiple --var options same key + (["--var", "a=1", "--var", "a=2", "noop"], {}, {"a": "2"}), + ( + ["--var-file", "foo.yaml", "--var", "key3.subkey1.id=id2", "noop"], + { + "foo.yaml": { + "key1": "val1", + "key2": "val2", + "key3": {"subkey1": {"id": "id1"}}, + } + }, + {"key1": "val1", "key2": "val2", "key3": {"subkey1": {"id": "id2"}}}, + ), + # one --var-file option + ( + ["--var-file", "foo.yaml", "noop"], + {"foo.yaml": {"key1": "val1", "key2": "val2"}}, + {"key1": "val1", "key2": "val2"}, + ), + # multiple --var-file option + ( + ["--var-file", "foo.yaml", "--var-file", "bar.yaml", "noop"], + { + "foo.yaml": {"key1": "parent_value1", "key2": "parent_value2"}, + "bar.yaml": {"key2": "child_value2", "key3": "child_value3"}, + }, + { + "key1": "parent_value1", + "key2": "child_value2", + "key3": "child_value3", + }, + ), + # mix of --var and --var-file + ( + ["--var-file", "foo.yaml", "--var", "key2=var2", "noop"], + {"foo.yaml": {"key1": "file1", "key2": "file2"}}, + {"key1": "file1", "key2": "var2"}, + ), + # multiple --var-file option, illustrating dictionaries not merged. + ( + ["--var-file", "foo.yaml", "--var-file", "bar.yaml", "noop"], + {"foo.yaml": {"key1": {"a": "b"}}, "bar.yaml": {"key1": {"c": "d"}}}, + {"key1": {"c": "d"}}, + ), + # multiple --var-file option, dictionaries merged. + ( + [ + "--merge-vars", + "--var-file", + "foo.yaml", + "--var-file", + "bar.yaml", + "noop", + ], + {"foo.yaml": {"key1": {"a": "b"}}, "bar.yaml": {"key1": {"c": "d"}}}, + {"key1": {"a": "b", "c": "d"}}, + ), + # multiple --var-file option, dictionaries merged, complex example. + ( + [ + "--merge-vars", + "--var-file", + "common.yaml", + "--var-file", + "dev.yaml", + "noop", + ], + { + "common.yaml": { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + } + }, + "dev.yaml": {"CommonTags": {"Environment": "dev"}}, + }, + { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Environment": "dev", + } + }, + ), + # multiple --var-file option, dictionaries merged, complex example, with overrides. + ( + [ + "--merge-vars", + "--var-file", + "common.yaml", + "--var-file", + "dev.yaml", + "noop", + ], + { + "common.yaml": { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Environment": "sandbox", + } + }, + "dev.yaml": {"CommonTags": {"Environment": "dev"}}, + }, + { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Environment": "dev", + } + }, + ), + # multiple --var-file option, dictionaries merged, complex example, with lists. + ( + [ + "--merge-vars", + "--var-file", + "common.yaml", + "--var-file", + "test.yaml", + "noop", + ], + { + "common.yaml": { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Envlist": ["sandbox", "dev"], + } + }, + "test.yaml": {"CommonTags": {"Envlist": ["test"]}}, + }, + { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Envlist": ["test"], + } + }, + ), + # multiple --var-file option, dictionaries merged, multiple levels. + ( + [ + "--merge-vars", + "--var-file", + "common.yaml", + "--var-file", + "test.yaml", + "noop", + ], + { + "common.yaml": {"a": {"b": {"c": "p", "d": "q"}}}, + "test.yaml": {"a": {"b": {"c": "r", "e": "s"}}}, + }, + {"a": {"b": {"c": "r", "d": "q", "e": "s"}}}, + ), + # a --var-file and --var combined. + ( + [ + "--merge-vars", + "--var-file", + "common.yaml", + "--var", + "CommonTags.Version=1.0.0", + "noop", + ], + { + "common.yaml": { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Envlist": ["sandbox", "dev"], + } + } + }, + { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Envlist": ["sandbox", "dev"], + "Version": "1.0.0", + } + }, + ), + # multiple --var-file and --var combined. + ( + [ + "--merge-vars", + "--var-file", + "common.yaml", + "--var-file", + "test.yaml", + "--var", + "CommonTags.Project=Unboxing", + "noop", + ], + { + "common.yaml": { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Envlist": ["sandbox", "dev"], + } + }, + "test.yaml": {"CommonTags": {"Project": "Boxing"}}, + }, + { + "CommonTags": { + "Organization": "Parts Unlimited", + "Department": "IT Operations", + "Envlist": ["sandbox", "dev"], + "Project": "Unboxing", + } + }, + ), + ], + ) + def test_user_variables(self, command, files, output): + @cli.command() + @click.pass_context + def noop(ctx): + click.echo(yaml.safe_dump(ctx.obj.get("user_variables"))) + + with self.runner.isolated_filesystem(): + for name, content in files.items(): + with open(name, "w") as fh: + yaml.safe_dump(content, fh) + + result = self.runner.invoke(cli, command) + + user_variables = yaml.safe_load(result.output) + assert result.exit_code == 0 + assert user_variables == output + + def test_validate_template_with_valid_template(self): + self.mock_stack_actions.validate.return_value = { + "Parameters": "Example", + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + + result_json = json.dumps({"Parameters": "Example"}, indent=4) + result = self.runner.invoke( + cli, ["--output", "json", "validate", "dev/vpc.yaml"] + ) + self.mock_stack_actions.validate.assert_called_with() + assert ( + result.output + == "Template mock-stack is valid. Template details:\n\n{}\n".format( + result_json + ) + ) + + def test_validate_template_with_invalid_template(self): + client_error = ClientError( + { + "Errors": { + "Message": "Unrecognized resource types", + "Code": "ValidationError", + } + }, + "ValidateTemplate", + ) + self.mock_stack_actions.validate.side_effect = client_error + + expected_result = str(client_error) + "\n" + result = self.runner.invoke( + cli, ["--output", "json", "validate", "dev/vpc.yaml"] + ) + assert expected_result in result.output.replace('"', "") + + def test_estimate_template_cost_with_browser(self): + self.mock_stack_actions.estimate_cost.return_value = { + "Url": "https://docs.sceptre-project.org", + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + + args = ["estimate-cost", "dev/vpc.yaml"] + with patch("webbrowser.open", return_value=None): # Do not open a web browser + result = self.runner.invoke(cli, args) + + self.mock_stack_actions.estimate_cost.assert_called_with() + + assert result.output == "{0}{1}".format( + "View the estimated cost for mock-stack at:\n", + "https://docs.sceptre-project.org\n\n", + ) + + def test_estimate_template_cost_with_no_browser(self): + client_error = ClientError( + { + "Errors": { + "Message": "No Browser", + "Code": "Error", + } + }, + "Webbrowser", + ) + self.mock_stack_actions.estimate_cost.side_effect = client_error + expected_result = "{}\n".format(client_error) + result = self.runner.invoke(cli, ["estimate-cost", "dev/vpc.yaml"]) + assert expected_result in result.output.replace('"', "") + + def test_lock_stack(self): + self.runner.invoke(cli, ["set-policy", "dev/vpc.yaml", "-b", "deny-all"]) + self.mock_config_reader.construct_stacks.assert_called_with() + self.mock_stack_actions.lock.assert_called_with() + + def test_unlock_stack(self): + self.runner.invoke(cli, ["set-policy", "dev/vpc.yaml", "-b", "allow-all"]) + self.mock_config_reader.construct_stacks.assert_called_with() + self.mock_stack_actions.unlock.assert_called_with() + + def test_set_policy_with_file_flag(self): + policy_file = "tests/fixtures/stack_policies/lock.json" + result = self.runner.invoke(cli, ["set-policy", "dev/vpc.yaml", policy_file]) + assert result.exit_code == 0 + + def test_describe_policy_with_existing_policy(self): + self.mock_stack_actions.get_policy.return_value = { + "dev/vpc": {"Statement": ["Body"]} + } + + result = self.runner.invoke( + cli, ["--output", "json", "describe", "policy", "dev/vpc.yaml"] + ) + assert result.exit_code == 0 + assert result.output == "{}\n".format( + json.dumps({"dev/vpc": {"Statement": ["Body"]}}, indent=4) + ) + + def test_list_group_resources(self): + response = { + "stack-name-1": { + "StackResources": [ + { + "LogicalResourceId": "logical-resource-id", + "PhysicalResourceId": "physical-resource-id", + } + ] + }, + "stack-name-2": { + "StackResources": [ + { + "LogicalResourceId": "logical-resource-id", + "PhysicalResourceId": "physical-resource-id", + } + ] + }, + } + self.mock_stack_actions.describe_resources.return_value = response + result = self.runner.invoke( + cli, ["--output", "yaml", "list", "resources", "dev"] + ) + + assert yaml.safe_load(result.output) == [response] + assert result.exit_code == 0 + + def test_list_stack_resources(self): + response = { + "StackResources": [ + { + "LogicalResourceId": "logical-resource-id", + "PhysicalResourceId": "physical-resource-id", + } + ] + } + self.mock_stack_actions.describe_resources.return_value = response + result = self.runner.invoke( + cli, ["--output", "yaml", "list", "resources", "dev/vpc.yaml"] + ) + assert yaml.safe_load(result.output) == [response] + assert result.exit_code == 0 + + @pytest.mark.parametrize( + "command,success,yes_flag,exit_code", + [ + ("create", True, True, 0), + ("create", False, True, 1), + ("create", True, False, 0), + ("create", False, False, 1), + ("delete", True, True, 0), + ("delete", False, True, 1), + ("delete", True, False, 0), + ("delete", False, False, 1), + ("update", True, True, 0), + ("update", False, True, 1), + ("update", True, False, 0), + ("update", False, False, 1), + ("launch", True, True, 0), + ("launch", False, True, 1), + ("launch", True, False, 0), + ("launch", False, False, 1), + ], + ) + def test_stack_commands(self, command, success, yes_flag, exit_code): + run_command = getattr(self.mock_stack_actions, command) + run_command.return_value = ( + StackStatus.COMPLETE if success else StackStatus.FAILED + ) + + kwargs = {"args": [command, "dev/vpc.yaml"]} + if yes_flag: + kwargs["args"].append("-y") + else: + kwargs["input"] = "y\n" + + result = self.runner.invoke(cli, **kwargs) + + run_command.assert_called_with() + assert result.exit_code == exit_code + + @pytest.mark.parametrize( + "command, ignore_dependencies", + [ + ("create", True), + ("create", False), + ("delete", True), + ("delete", False), + ], + ) + def test_ignore_dependencies_commands(self, command, ignore_dependencies): + args = [command, "dev/vpc.yaml", "cs-1", "-y"] + if ignore_dependencies: + args.insert(0, "--ignore-dependencies") + result = self.runner.invoke(cli, args) + assert result.exit_code == 0 + + @pytest.mark.parametrize( + "command,yes_flag", + [ + ("create", True), + ("create", False), + ("delete", True), + ("delete", False), + ("execute", True), + ("execute", False), + ], + ) + def test_change_set_commands(self, command, yes_flag): + stack_command = command + "_change_set" + + kwargs = {"args": [command, "dev/vpc.yaml", "cs1"]} + if yes_flag: + kwargs["args"].append("-y") + else: + kwargs["input"] = "y\n" + + result = self.runner.invoke(cli, **kwargs) + + getattr(self.mock_stack_actions, stack_command).assert_called_with("cs1") + assert result.exit_code == 0 + + @pytest.mark.parametrize("verbose_flag,", [(False), (True)]) + def test_describe_change_set(self, verbose_flag): + response = { + "VerboseProperty": "VerboseProperty", + "ChangeSetName": "ChangeSetName", + "CreationTime": "CreationTime", + "ExecutionStatus": "ExecutionStatus", + "StackName": "StackName", + "Status": "Status", + "StatusReason": "StatusReason", + "Changes": [ + { + "ResourceChange": { + "Action": "Action", + "LogicalResourceId": "LogicalResourceId", + "PhysicalResourceId": "PhysicalResourceId", + "Replacement": "Replacement", + "ResourceType": "ResourceType", + "Scope": "Scope", + "VerboseProperty": "VerboseProperty", + } + } + ], + } + args = ["describe", "change-set", "region/vpc.yaml", "cs1"] + if verbose_flag: + args.append("-v") + + self.mock_stack_actions.describe_change_set.return_value = response + result = self.runner.invoke(cli, args) + if not verbose_flag: + del response["VerboseProperty"] + del response["Changes"][0]["ResourceChange"]["VerboseProperty"] + assert yaml.safe_load(result.output) == response + assert result.exit_code == 0 + + def test_list_change_sets_with_200(self): + self.mock_stack_actions.list_change_sets.return_value = {"ChangeSets": "Test"} + result = self.runner.invoke(cli, ["list", "change-sets", "dev/vpc.yaml"]) + assert result.exit_code == 0 + assert yaml.safe_load(result.output) == {"ChangeSets": "Test"} + + def test_list_change_sets_without_200(self): + response = {"ChangeSets": "Test"} + self.mock_stack_actions.list_change_sets.return_value = response + + result = self.runner.invoke( + cli, ["--output", "json", "list", "change-sets", "dev/vpc.yaml"] + ) + assert result.exit_code == 0 + assert yaml.safe_load(result.output) == response + + def test_list_outputs_json(self): + outputs = {"OutputKey": "Key", "OutputValue": "Value"} + self.mock_stack_actions.describe_outputs.return_value = outputs + result = self.runner.invoke( + cli, ["--output", "json", "list", "outputs", "dev/vpc.yaml"] + ) + assert result.exit_code == 0 + assert json.loads(result.output) == [outputs] + + def test_list_outputs_yaml(self): + outputs = {"OutputKey": "Key", "OutputValue": "Value"} + self.mock_stack_actions.describe_outputs.return_value = outputs + result = self.runner.invoke( + cli, ["--output", "yaml", "list", "outputs", "dev/vpc.yaml"] + ) + assert result.exit_code == 0 + expected_output = "---\n- OutputKey: Key\n OutputValue: Value\n\n" + assert result.output == expected_output + + def test_list_outputs_text(self): + outputs = {"StackName": [{"OutputKey": "Key", "OutputValue": "Value"}]} + self.mock_stack_actions.describe_outputs.return_value = outputs + result = self.runner.invoke( + cli, ["--output", "text", "list", "outputs", "dev/vpc.yaml"] + ) + assert result.exit_code == 0 + expected_output = "StackOutputKeyOutputValue\n\nStackNameKeyValue\n" + assert result.output.replace(" ", "") == expected_output + + def test_list_outputs_with_export(self): + outputs = {"stack": [{"OutputKey": "Key", "OutputValue": "Value"}]} + self.mock_stack_actions.describe_outputs.return_value = outputs + result = self.runner.invoke( + cli, ["list", "outputs", "dev/vpc.yaml", "-e", "envvar"] + ) + assert result.exit_code == 0 + assert result.output == "export SCEPTRE_Key='Value'\n" + + @pytest.mark.parametrize( + "path,output_format,expected_output", + [ + ("dev/vpc.yaml", "yaml", "---\nmock-stack.yaml: mock-stack-external\n\n"), + ("dev/vpc.yaml", "text", "---\nmock-stack.yaml: mock-stack-external\n\n"), + ( + "dev/vpc.yaml", + "json", + '{\n "mock-stack.yaml": "mock-stack-external"\n}\n', + ), + ("dev", "yaml", "---\nmock-stack.yaml: mock-stack-external\n\n"), + ], + ) + def test_list_stacks(self, path, output_format, expected_output): + result = self.runner.invoke( + cli, ["--output", output_format, "list", "stacks", path] + ) + assert result.exit_code == 0 + assert result.stdout == expected_output + + def test_status_with_group(self): + self.mock_stack_actions.get_status.return_value = {"stack": "status"} + + result = self.runner.invoke(cli, ["--output", "json", "status", "dev"]) + assert result.exit_code == 0 + assert ( + result.output + == '{\n "mock-stack": {\n "stack": "status"\n }\n}\n' + ) + + def test_status_with_stack(self): + self.mock_stack_actions.get_status.return_value = "status" + result = self.runner.invoke(cli, ["status", "dev/vpc.yaml"]) + assert result.exit_code == 0 + assert result.output == '{\n "mock-stack": "status"\n}\n' + + def test_new_project_non_existant(self): + with self.runner.isolated_filesystem(): + project_path = os.path.abspath("./example") + config_dir = os.path.join(project_path, "config") + template_dir = os.path.join(project_path, "templates") + region = "test-region" + os.environ["AWS_DEFAULT_REGION"] = region + defaults = {"project_code": "example", "region": region} + + result = self.runner.invoke(cli, ["new", "project", "example"]) + assert not result.exception + assert os.path.isdir(config_dir) + assert os.path.isdir(template_dir) + + with open(os.path.join(config_dir, "config.yaml")) as config_file: + config = yaml.safe_load(config_file) + + assert config == defaults + + def test_new_project_already_exist(self): + with self.runner.isolated_filesystem(): + project_path = os.path.abspath("./example") + config_dir = os.path.join(project_path, "config") + template_dir = os.path.join(project_path, "templates") + existing_config = {"Test": "Test"} + + os.mkdir(project_path) + os.mkdir(config_dir) + os.mkdir(template_dir) + + config_filepath = os.path.join(config_dir, "config.yaml") + with open(config_filepath, "w") as config_file: + yaml.dump(existing_config, config_file) + + result = self.runner.invoke(cli, ["new", "project", "example"]) + assert result.exit_code == 1 + assert result.output == '"Folder \\"example\\" already exists."\n' + assert os.path.isdir(config_dir) + assert os.path.isdir(template_dir) + + with open(os.path.join(config_dir, "config.yaml")) as config_file: + config = yaml.safe_load(config_file) + assert existing_config == config + + def test_new_project_another_exception(self): + with self.runner.isolated_filesystem(): + patcher_mkdir = patch("sceptre.cli.new.os.mkdir") + mock_mkdir = patcher_mkdir.start() + mock_mkdir.side_effect = OSError(errno.EINVAL) + result = self.runner.invoke(cli, ["new", "project", "example"]) + mock_mkdir = patcher_mkdir.stop() + assert str(result.exception) == str(OSError(errno.EINVAL)) + + @pytest.mark.parametrize( + "stack_group,config_structure,stdin,result", + [ + ("A", {"": {}}, "y\nA\nA\n", {"project_code": "A", "region": "A"}), + ("A", {"": {"project_code": "top", "region": "top"}}, "y\n\n\n", {}), + ( + "A", + {"": {"project_code": "top", "region": "top"}}, + "y\nA\nA\n", + {"project_code": "A", "region": "A"}, + ), + ( + "A/A", + { + "": {"project_code": "top", "region": "top"}, + "A": {"project_code": "A", "region": "A"}, + }, + "y\nA/A\nA/A\n", + {"project_code": "A/A", "region": "A/A"}, + ), + ( + "A/A", + { + "": {"project_code": "top", "region": "top"}, + "A": {"project_code": "A", "region": "A"}, + }, + "y\nA\nA\n", + {}, + ), + ], + ) + def test_create_new_stack_group_folder( + self, stack_group, config_structure, stdin, result + ): + with self.runner.isolated_filesystem(): + project_path = os.path.abspath("./example") + config_dir = os.path.join(project_path, "config") + os.makedirs(config_dir) + + stack_group_dir = os.path.join(project_path, "config", stack_group) + for stack_group_path, config in config_structure.items(): + path = os.path.join(config_dir, stack_group_path) + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + filepath = os.path.join(path, "config.yaml") + with open(filepath, "w") as config_file: + yaml.safe_dump(config, stream=config_file, default_flow_style=False) + + os.chdir(project_path) + + cmd_result = self.runner.invoke( + cli, ["new", "group", stack_group], input=stdin + ) + + if result: + with open(os.path.join(stack_group_dir, "config.yaml")) as config_file: + config = yaml.safe_load(config_file) + assert config == result + else: + assert cmd_result.output.endswith( + "No config.yaml file needed - covered by parent config.\n" + ) + + def test_new_stack_group_folder_with_existing_folder(self): + with self.runner.isolated_filesystem(): + project_path = os.path.abspath("./example") + config_dir = os.path.join(project_path, "config") + stack_group_dir = os.path.join(config_dir, "A") + + os.makedirs(stack_group_dir) + os.chdir(project_path) + + cmd_result = self.runner.invoke(cli, ["new", "group", "A"], input="y\n\n\n") + + assert cmd_result.output.startswith( + "StackGroup path exists. " "Do you want initialise config.yaml?" + ) + with open(os.path.join(stack_group_dir, "config.yaml")) as config_file: + config = yaml.safe_load(config_file) + assert config == {"project_code": "", "region": ""} + + def test_new_stack_group_folder_with_another_exception(self): + with self.runner.isolated_filesystem(): + project_path = os.path.abspath("./example") + config_dir = os.path.join(project_path, "config") + stack_group_dir = os.path.join(config_dir, "A") + + os.makedirs(stack_group_dir) + os.chdir(project_path) + patcher_mkdir = patch("sceptre.cli.new.os.mkdir") + mock_mkdir = patcher_mkdir.start() + mock_mkdir.side_effect = OSError(errno.EINVAL) + result = self.runner.invoke(cli, ["new", "group", "A"]) + mock_mkdir = patcher_mkdir.stop() + assert str(result.exception) == str(OSError(errno.EINVAL)) + + @pytest.mark.parametrize( + "cli_module,command,output_format,no_colour", + [ + ("describe", ["describe", "change-set", "somepath", "cs1"], "yaml", True), + ("describe", ["describe", "change-set", "somepath", "cs1"], "json", False), + ("describe", ["describe", "policy", "somepolicy"], "yaml", True), + ("describe", ["describe", "policy", "somepolicy"], "json", False), + ], + ) + def test_write_output_format_flags( + self, cli_module, command, output_format, no_colour + ): + no_colour_flag = ["--no-colour"] if no_colour else [] + output_format_flag = ["--output", output_format] + args = output_format_flag + no_colour_flag + command + + with patch("sceptre.cli." + cli_module + ".write") as mock_write: + self.runner.invoke(cli, args) + mock_write.assert_called() + for call in mock_write.call_args_list: + args, _ = call + assert args[1] == output_format + assert args[2] == no_colour + + def test_setup_logging_with_debug(self): + logger = setup_logging(True, False) + assert logger.getEffectiveLevel() == logging.DEBUG + assert logging.getLogger("botocore").getEffectiveLevel() == logging.INFO + + # Silence logging for the rest of the tests + logger.setLevel(logging.CRITICAL) + + def test_setup_logging_without_debug(self): + logger = setup_logging(False, False) + assert logger.getEffectiveLevel() == logging.INFO + assert logging.getLogger("botocore").getEffectiveLevel() == logging.CRITICAL + + # Silence logging for the rest of the tests + logger.setLevel(logging.CRITICAL) + + @patch("sceptre.cli.click.echo") + @pytest.mark.parametrize( + "output_format,no_colour,expected_output", + [ + ("json", True, '{\n "stack": "CREATE_COMPLETE"\n}'), + ("json", False, '{\n "stack": "\x1b[32mCREATE_COMPLETE\x1b[0m"\n}'), + ("yaml", True, "---\nstack: CREATE_COMPLETE\n"), + ("yaml", False, "---\nstack: \x1b[32mCREATE_COMPLETE\x1b[0m\n"), + ], + ) + def test_write_formats(self, mock_echo, output_format, no_colour, expected_output): + write({"stack": "CREATE_COMPLETE"}, output_format, no_colour) + mock_echo.assert_called_once_with(expected_output) + + @patch("sceptre.cli.click.echo") + def test_write_status_with_colour(self, mock_echo): + write("stack: CREATE_COMPLETE", no_colour=False) + mock_echo.assert_called_once_with( + '{\n "stack": "\x1b[32mCREATE_COMPLETE\x1b[0m"\n}' + ) + + @patch("sceptre.cli.click.echo") + def test_write_status_without_colour(self, mock_echo): + write("stack: CREATE_COMPLETE", no_colour=True) + mock_echo.assert_called_once_with('{\n "stack": "CREATE_COMPLETE"\n}') + + @patch("sceptre.cli.helpers.StackStatusColourer.colour") + @patch("sceptre.cli.helpers.logging.Formatter.format") + def test_ColouredFormatter_format_with_string(self, mock_format, mock_colour): + mock_format.return_value = sentinel.response + mock_colour.return_value = sentinel.coloured_response + coloured_formatter = ColouredFormatter() + response = coloured_formatter.format("string") + mock_format.assert_called_once_with("string") + mock_colour.assert_called_once_with(sentinel.response) + assert response == sentinel.coloured_response + + def test_CustomJsonEncoder_with_non_json_serialisable_object(self): + encoder = CustomJsonEncoder() + response = encoder.encode(datetime.datetime(2016, 5, 3)) + assert response == '"2016-05-03 00:00:00"' + + def test_diff_command__diff_type_is_deepdiff__passes_deepdiff_stack_differ_to_actions( + self, + ): + self.runner.invoke(cli, "diff -t deepdiff dev/vpc.yaml") + differ_used = self.mock_stack_actions.diff.call_args[0][0] + assert isinstance(differ_used, DeepDiffStackDiffer) + + def test_diff_command__diff_type_is_difflib__passes_difflib_stack_differ_to_actions( + self, + ): + self.runner.invoke(cli, "diff -t difflib dev/vpc.yaml") + differ_used = self.mock_stack_actions.diff.call_args[0][0] + assert isinstance(differ_used, DifflibStackDiffer) + + self.runner.invoke(cli, "diff stacks", catch_exceptions=False) + + def test_diff_command__stack_diffs_have_differences__returns_0(self): + stacks = {deepcopy(self.mock_stack) for _ in range(3)} + stack_name_iterator = iter(["first", "second", "third"]) + + def fake_diff(differ): + name = next(stack_name_iterator) + return StackDiff( + stack_name=name, + template_diff=DeepDiff("I'm", "different"), + config_diff=DeepDiff("same", "same"), + is_deployed=True, + generated_config=None, + generated_template=None, + ) + + self.mock_stack_actions.diff.side_effect = fake_diff + self.mock_config_reader.construct_stacks.return_value = (stacks, stacks) + + result = self.runner.invoke(cli, "diff stacks", catch_exceptions=False) + assert result.exit_code == 0 + + def test_diff_command__no_differences__returns_0(self): + stacks = {deepcopy(self.mock_stack) for _ in range(3)} + stack_name_iterator = iter(["first", "second", "third"]) + + def fake_diff(differ): + name = next(stack_name_iterator) + return StackDiff( + stack_name=name, + template_diff=DeepDiff("same", "same"), + config_diff=DeepDiff("same", "same"), + is_deployed=True, + generated_config=None, + generated_template=None, + ) + + self.mock_stack_actions.diff.side_effect = fake_diff + self.mock_config_reader.construct_stacks.return_value = (stacks, stacks) + + result = self.runner.invoke(cli, "diff stacks", catch_exceptions=False) + assert result.exit_code == 0 + + @pytest.mark.parametrize(["bar"], [("**********",), ("----------",)]) + def test_diff_command__bars_are_all_full_width_of_output(self, bar): + stacks = {deepcopy(self.mock_stack) for _ in range(3)} + stack_name_iterator = iter(["first", "second", "third"]) + + def fake_diff(differ): + name = next(stack_name_iterator) + return StackDiff( + stack_name=name, + template_diff=DeepDiff("same", "same"), + config_diff=DeepDiff("same", "same"), + is_deployed=True, + generated_config=None, + generated_template=None, + ) + + self.mock_stack_actions.diff.side_effect = fake_diff + self.mock_config_reader.construct_stacks.return_value = (stacks, stacks) + + result = self.runner.invoke(cli, "diff stacks", catch_exceptions=False) + output_lines = result.stdout.splitlines() + max_line_length = len(max(output_lines, key=len)) + star_bars = [line for line in output_lines if bar in line] + assert all(len(line) == max_line_length for line in star_bars) + + @pytest.mark.parametrize( + "input,expected_output", + [ + ( + {"a_dict": '{"with_embedded":"json"}'}, + {"a_dict": {"with_embedded": "json"}}, + ), + ( + {"a_dict": ['{"with_embedded":"json"}']}, + {"a_dict": [{"with_embedded": "json"}]}, + ), + ], + ) + def test_deserialize_json_properties(self, input, expected_output): + output = deserialize_json_properties(input) + assert output == expected_output + + def test_drift_detect(self): + self.mock_stack_actions.drift_detect.return_value = { + "StackId": "fake-stack-id", + "StackDriftDetectionId": "3fb76910-f660-11eb-80ac-0246f7a6da62", + "StackDriftStatus": "IN_SYNC", + "DetectionStatus": "DETECTION_COMPLETE", + "DriftedStackResourceCount": 0, + } + result = self.runner.invoke(cli, ["drift", "detect", "dev/vpc.yaml"]) + assert result.exit_code == 0 + assert result.output == ( + "---\n" + "mock-stack-external:\n" + " DetectionStatus: DETECTION_COMPLETE\n" + " DriftedStackResourceCount: 0\n" + " StackDriftDetectionId: 3fb76910-f660-11eb-80ac-0246f7a6da62\n" + " StackDriftStatus: IN_SYNC\n" + " StackId: fake-stack-id\n\n" + ) + + def test_drift_show(self): + self.mock_stack_actions.drift_show.return_value = ( + "DETECTION_COMPLETE", + {"some": "json"}, + ) + result = self.runner.invoke(cli, ["drift", "show", "dev/vpc.yaml"]) + assert result.exit_code == 0 + assert result.output == "---\nmock-stack-external:\n some: json\n\n" diff --git a/tests/test_cli/test_launch.py b/tests/test_cli/test_launch.py new file mode 100644 index 000000000..584516979 --- /dev/null +++ b/tests/test_cli/test_launch.py @@ -0,0 +1,217 @@ +import functools +import itertools +from collections import defaultdict +from typing import Optional, List, Set +from unittest.mock import create_autospec, Mock + +import pytest + +from sceptre.cli.launch import Launcher +from sceptre.cli.prune import Pruner +from sceptre.context import SceptreContext +from sceptre.exceptions import DependencyDoesNotExistError +from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack +from sceptre.stack_status import StackStatus + + +class FakePlan(SceptrePlan): + def __init__( + self, + context: SceptreContext, + command_stacks: Set[Stack], + all_stacks: Set[Stack], + statuses_to_return: dict, + ): + self.context = context + self.command = None + self.reverse = None + self.launch_order: Optional[List[Set[Stack]]] = None + + self.all_stacks = all_stacks + self.command_stacks = command_stacks + self.statuses_to_return = statuses_to_return + + self.executions = [] + + def _execute(self, *args): + self.executions.append((self.command, self.launch_order.copy(), args)) + return {stack: self.statuses_to_return[stack] for stack in self} + + def _generate_launch_order(self, reverse=False) -> List[Set[Stack]]: + launch_order = [self.command_stacks] + if self.context.ignore_dependencies: + return launch_order + + all_stacks = list(self.all_stacks) + + for start_index in range(0, len(all_stacks), 2): + chunk = { + stack + for stack in all_stacks[start_index : start_index + 2] + if stack not in self.command_stacks + and self._has_dependency_on_a_command_stack(stack) + } + if len(chunk): + launch_order.append(chunk) + + return launch_order + + @functools.lru_cache() + def _has_dependency_on_a_command_stack(self, stack): + if len(self.command_stacks.intersection(stack.dependencies)): + return True + + for dependency in stack.dependencies: + if self._has_dependency_on_a_command_stack(dependency): + return True + + return False + + +class TestLauncher: + def setup_method(self, test_method): + self.plans: List[FakePlan] = [] + + self.context = SceptreContext( + project_path="project", + command_path="my-test-group", + ) + self.cloned_context = self.context.clone() + # Since contexts don't have a __eq__ method, you can't assert easily off the result of + # clone without some hijinks. + self.context = Mock( + wraps=self.context, + **{"clone.return_value": self.cloned_context, "ignore_dependencies": False}, + ) + + self.all_stacks = [ + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + ] + for index, stack in enumerate(self.all_stacks): + stack.name = f"stacks/stack-{index}.yaml" + + self.command_stacks = list(self.all_stacks) + + self.statuses_to_return = defaultdict(lambda: StackStatus.COMPLETE) + + self.fake_pruner = Mock(spec=Pruner, **{"prune.return_value": 0}) + + self.plan_factory = create_autospec(SceptrePlan) + self.plan_factory.side_effect = self.fake_plan_factory + self.pruner_factory = create_autospec(Pruner) + self.pruner_factory.return_value = self.fake_pruner + + self.launcher = Launcher(self.context, self.plan_factory, self.pruner_factory) + + def fake_plan_factory(self, sceptre_context): + fake_plan = FakePlan( + sceptre_context, + set(self.command_stacks), + set(self.all_stacks), + self.statuses_to_return, + ) + self.plans.append(fake_plan) + return fake_plan + + def get_executed_stacks(self, plan_number: int): + launch_order = self.plans[plan_number].executions[0][1] + return list(itertools.chain.from_iterable(launch_order)) + + def test_launch__launches_stacks_that_are_neither_ignored_nor_obsolete(self): + assert all(not s.ignore and not s.obsolete for s in self.all_stacks) + self.command_stacks = self.all_stacks + self.launcher.launch(True) + launched_stacks = set(self.get_executed_stacks(0)) + expected_stacks = set(self.all_stacks) + assert expected_stacks == launched_stacks + assert self.plans[0].executions[0][0] == "launch" + assert len(self.plans[0].executions) == 1 + + def test_launch__prune__no_obsolete_stacks__does_not_delete_any_stacks(self): + assert all(not s.obsolete for s in self.all_stacks) + self.launcher.launch(True) + assert len(self.plans) == 1 + assert self.plans[0].executions[0][0] == "launch" + + def test_launch__prune__instantiates_and_invokes_pruner(self): + self.launcher.launch(True) + self.fake_pruner.prune.assert_any_call() + + def test_launch__no_prune__obsolete_stacks__does_not_delete_any_stacks(self): + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + + self.launcher.launch(False) + assert len(self.plans) == 1 + assert self.plans[0].executions[0][0] == "launch" + + @pytest.mark.parametrize( + "prune", [pytest.param(True, id="prune"), pytest.param(False, id="no prune")] + ) + def test_launch__returns_0(self, prune): + assert all(not s.ignore and not s.obsolete for s in self.all_stacks) + result = self.launcher.launch(prune) + + assert result == 0 + + def test_launch__does_not_launch_stacks_that_should_be_excluded(self): + self.all_stacks[4].ignore = True + self.all_stacks[5].obsolete = True + + self.launcher.launch(True) + + launched_stacks = set(self.get_executed_stacks(0)) + expected_stacks = {s for i, s in enumerate(self.all_stacks) if i not in (4, 5)} + assert expected_stacks == launched_stacks + assert self.plans[0].executions[0][0] == "launch" + + def test_launch__prune__stack_with_dependency_marked_obsolete__raises_dependency_does_not_exist_error( + self, + ): + self.all_stacks[0].obsolete = True + self.all_stacks[1].dependencies.append(self.all_stacks[0]) + + self.command_stacks = [self.all_stacks[0]] + + with pytest.raises(DependencyDoesNotExistError): + self.launcher.launch(True) + + def test_launch__prune__ignore_dependencies__stack_with_dependency_marked_obsolete__raises_no_error( + self, + ): + self.all_stacks[0].obsolete = True + self.all_stacks[1].dependencies.append(self.all_stacks[0]) + + self.command_stacks = [self.all_stacks[0]] + self.context.ignore_dependencies = True + self.launcher.launch(True) + + def test_launch__no_prune__does_not_raise_error(self): + self.all_stacks[0].obsolete = True + self.all_stacks[1].dependencies.append(self.all_stacks[0]) + self.launcher.launch(False) + + def test_launch__stacks_are_pruned__delete_and_deploy_actions_succeed__returns_0( + self, + ): + self.all_stacks[0].obsolete = True + + code = self.launcher.launch(True) + assert code == 0 + + def test_launch__pruner_returns_nonzero__returns_nonzero(self): + self.fake_pruner.prune.return_value = 99 + + code = self.launcher.launch(True) + assert code == 99 + + def test_launch__deploy_action_fails__returns_nonzero(self): + self.statuses_to_return[self.all_stacks[3]] = StackStatus.FAILED + code = self.launcher.launch(False) + assert code != 0 diff --git a/tests/test_cli/test_prune.py b/tests/test_cli/test_prune.py new file mode 100644 index 000000000..2f8865d1c --- /dev/null +++ b/tests/test_cli/test_prune.py @@ -0,0 +1,209 @@ +import functools +import itertools +from collections import defaultdict +from typing import Set, Optional, List +from unittest.mock import Mock, create_autospec + +import pytest + +from sceptre.cli.prune import Pruner, PATH_FOR_WHOLE_PROJECT +from sceptre.context import SceptreContext +from sceptre.exceptions import CannotPruneStackError +from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack +from sceptre.stack_status import StackStatus + + +class FakePlan(SceptrePlan): + def __init__( + self, + context: SceptreContext, + command_stacks: Set[Stack], + all_stacks: Set[Stack], + statuses_to_return: dict, + ): + self.context = context + self.command = None + self.reverse = None + self.launch_order: Optional[List[Set[Stack]]] = None + + self.all_stacks = self.graph = all_stacks + self.command_stacks = command_stacks + self.statuses_to_return = statuses_to_return + + self.executions = [] + + def _execute(self, *args): + self.executions.append((self.command, self.launch_order.copy(), args)) + return {stack: self.statuses_to_return[stack] for stack in self} + + def _generate_launch_order(self, reverse=False) -> List[Set[Stack]]: + launch_order = [self.command_stacks] + if self.context.ignore_dependencies: + return launch_order + + all_stacks = list(self.all_stacks) + + for start_index in range(0, len(all_stacks), 2): + chunk = { + stack + for stack in all_stacks[start_index : start_index + 2] + if stack not in self.command_stacks + and self._has_dependency_on_a_command_stack(stack) + } + if len(chunk): + launch_order.append(chunk) + + return launch_order + + @functools.lru_cache() + def _has_dependency_on_a_command_stack(self, stack): + if len(self.command_stacks.intersection(stack.dependencies)): + return True + + for dependency in stack.dependencies: + if self._has_dependency_on_a_command_stack(dependency): + return True + + return False + + +class TestPruner: + def setup_method(self, test_method): + self.plans: List[FakePlan] = [] + + self.context = SceptreContext( + project_path="project", + command_path=PATH_FOR_WHOLE_PROJECT, + ) + + self.all_stacks = [ + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + ] + for index, stack in enumerate(self.all_stacks): + stack.name = f"stacks/stack-{index}.yaml" + + self.command_stacks = [self.all_stacks[2], self.all_stacks[4]] + + self.statuses_to_return = defaultdict(lambda: StackStatus.COMPLETE) + + self.plan_factory = create_autospec(SceptrePlan) + self.plan_factory.side_effect = self.fake_plan_factory + + self.pruner = Pruner(self.context, self.plan_factory) + + def fake_plan_factory(self, sceptre_context): + fake_plan = FakePlan( + sceptre_context, + set(self.command_stacks), + set(self.all_stacks), + self.statuses_to_return, + ) + self.plans.append(fake_plan) + return fake_plan + + @property + def executed_stacks(self): + assert len(self.plans) == 1 + launch_order = self.plans[0].executions[0][1] + return list(itertools.chain.from_iterable(launch_order)) + + def test_prune__no_obsolete_stacks__returns_zero(self): + code = self.pruner.prune() + assert code == 0 + + def test_prune__no_obsolete_stacks__does_not_call_command_on_plan(self): + self.pruner.prune() + assert len(self.plans[0].executions) == 0 + + def test_prune__whole_project__obsolete_stacks__deletes_all_obsolete_stacks(self): + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + + self.pruner.prune() + + assert self.plans[0].executions[0][0] == "delete" + assert set(self.executed_stacks) == {self.all_stacks[4], self.all_stacks[5]} + + def test_prune__command_path__obsolete_stacks__deletes_only_obsolete_stacks_on_path( + self, + ): + self.all_stacks[4].obsolete = True # On command path + self.all_stacks[5].obsolete = True # not on command path + self.context.command_path = "my/command/path" + self.pruner.prune() + + assert self.plans[0].executions[0][0] == "delete" + assert set(self.executed_stacks) == {self.all_stacks[4]} + + def test_prune__obsolete_stacks__returns_zero(self): + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + + code = self.pruner.prune() + assert code == 0 + + def test_prune__obsolete_stacks_depend_on_other_obsolete_stacks__deletes_only_obsolete_stacks( + self, + ): + self.all_stacks[1].obsolete = True + self.all_stacks[3].obsolete = True + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + self.all_stacks[3].dependencies.append(self.all_stacks[1]) + self.all_stacks[4].dependencies.append(self.all_stacks[3]) + self.all_stacks[5].dependencies.append(self.all_stacks[3]) + + self.pruner.prune() + + assert self.plans[0].executions[0][0] == "delete" + assert set(self.executed_stacks) == { + self.all_stacks[1], + self.all_stacks[3], + self.all_stacks[4], + self.all_stacks[5], + } + + def test_prune__non_obsolete_stacks_depend_on_obsolete_stacks__raises_cannot_prune_stack_error( + self, + ): + self.all_stacks[1].obsolete = True + self.all_stacks[3].obsolete = False + self.all_stacks[4].obsolete = False + self.all_stacks[5].obsolete = False + self.all_stacks[3].dependencies.append(self.all_stacks[1]) + self.all_stacks[4].dependencies.append(self.all_stacks[3]) + self.all_stacks[5].dependencies.append(self.all_stacks[3]) + + with pytest.raises(CannotPruneStackError): + self.pruner.prune() + + def test_prune__non_obsolete_stacks_depend_on_obsolete_stacks__ignore_dependencies__deletes_obsolete_stacks( + self, + ): + self.all_stacks[1].obsolete = True + self.all_stacks[3].obsolete = False + self.all_stacks[4].obsolete = False + self.all_stacks[5].obsolete = False + self.all_stacks[3].dependencies.append(self.all_stacks[1]) + self.all_stacks[4].dependencies.append(self.all_stacks[3]) + self.all_stacks[5].dependencies.append(self.all_stacks[3]) + self.context.ignore_dependencies = True + self.pruner.prune() + + assert self.plans[0].executions[0][0] == "delete" + assert set(self.executed_stacks) == { + self.all_stacks[1], + } + + def test_prune__delete_action_fails__returns_nonzero(self): + self.all_stacks[1].obsolete = True + self.statuses_to_return[self.all_stacks[1]] = StackStatus.FAILED + + code = self.pruner.prune() + assert code != 0 diff --git a/tests/test_config_reader.py b/tests/test_config_reader.py index 6f8ae6be9..1918d5f16 100644 --- a/tests/test_config_reader.py +++ b/tests/test_config_reader.py @@ -1,33 +1,39 @@ # -*- coding: utf-8 -*- +import errno import os -from mock import patch, sentinel +from unittest.mock import patch, sentinel, MagicMock + import pytest import yaml -import errno - -from sceptre.context import SceptreContext -from sceptre.exceptions import DependencyDoesNotExistError -from sceptre.exceptions import VersionIncompatibleError -from sceptre.exceptions import ConfigFileNotFoundError -from sceptre.exceptions import InvalidSceptreDirectoryError -from sceptre.exceptions import InvalidConfigFileError - -from freezegun import freeze_time from click.testing import CliRunner +from freezegun import freeze_time + from sceptre.config.reader import ConfigReader +from sceptre.context import SceptreContext +from sceptre.exceptions import ( + DependencyDoesNotExistError, + VersionIncompatibleError, + ConfigFileNotFoundError, + InvalidSceptreDirectoryError, + InvalidConfigFileError, +) class TestConfigReader(object): @patch("sceptre.config.reader.ConfigReader._check_valid_project_path") def setup_method(self, test_method, mock_check_valid_project_path): self.runner = CliRunner() - self.test_project_path = os.path.join( - os.getcwd(), "tests", "fixtures" - ) + self.test_project_path = os.path.join(os.getcwd(), "tests", "fixtures") self.context = SceptreContext( project_path=self.test_project_path, - command_path="A" + command_path="A", + command_params={ + "yes": True, + "path": "A.yaml", + "prune": False, + "disable_rollback": None, + }, ) def test_config_reader_correctly_initialised(self): @@ -43,7 +49,7 @@ def create_project(self): Creates a new random temporary directory with a config subdirectory """ with self.runner.isolated_filesystem(): - project_path = os.path.abspath('./example') + project_path = os.path.abspath("./example") config_dir = os.path.join(project_path, "config") os.makedirs(config_dir) return (project_path, config_dir) @@ -61,22 +67,17 @@ def write_config(self, abs_path, config): if exc.errno != errno.EEXIST: raise - with open(abs_path, 'w') as config_file: - yaml.safe_dump( - config, stream=config_file, default_flow_style=False - ) - - @pytest.mark.parametrize("filepaths,target", [ - ( - ["A/1.yaml"], "A/1.yaml" - ), - ( - ["A/1.yaml", "A/B/1.yaml"], "A/B/1.yaml" - ), - ( - ["A/1.yaml", "A/B/1.yaml", "A/B/C/1.yaml"], "A/B/C/1.yaml" - ) - ]) + with open(abs_path, "w") as config_file: + yaml.safe_dump(config, stream=config_file, default_flow_style=False) + + @pytest.mark.parametrize( + "filepaths,target", + [ + (["A/1.yaml"], "A/1.yaml"), + (["A/1.yaml", "A/B/1.yaml"], "A/B/1.yaml"), + (["A/1.yaml", "A/B/1.yaml", "A/B/C/1.yaml"], "A/B/C/1.yaml"), + ], + ) def test_read_reads_config_file(self, filepaths, target): project_path, config_dir = self.create_project() @@ -91,37 +92,94 @@ def test_read_reads_config_file(self, filepaths, target): assert config == { "project_path": project_path, "stack_group_path": os.path.split(target)[0], - "filepath": target + "filepath": target, } + def test_read_nested_configs(self): + with self.runner.isolated_filesystem(): + project_path = os.path.abspath("./example") + config_dir = os.path.join(project_path, "config") + stack_group_dir_a = os.path.join(config_dir, "A") + stack_group_dir_b = os.path.join(stack_group_dir_a, "B") + stack_group_dir_c = os.path.join(stack_group_dir_b, "C") + + os.makedirs(stack_group_dir_c) + config_filename = "config.yaml" + + config_a = {"keyA": "A", "shared": "A"} + with open( + os.path.join(stack_group_dir_a, config_filename), "w" + ) as config_file: + yaml.safe_dump(config_a, stream=config_file, default_flow_style=False) + + config_b = {"keyB": "B", "parent": "{{ keyA }}", "shared": "B"} + with open( + os.path.join(stack_group_dir_b, config_filename), "w" + ) as config_file: + yaml.safe_dump(config_b, stream=config_file, default_flow_style=False) + + config_c = {"keyC": "C", "parent": "{{ keyB }}", "shared": "C"} + with open( + os.path.join(stack_group_dir_c, config_filename), "w" + ) as config_file: + yaml.safe_dump(config_c, stream=config_file, default_flow_style=False) + + self.context.project_path = project_path + reader = ConfigReader(self.context) + + config_a = reader.read("A/config.yaml") + + assert config_a == { + "project_path": project_path, + "stack_group_path": "A", + "keyA": "A", + "shared": "A", + } + + config_b = reader.read("A/B/config.yaml") + + assert config_b == { + "project_path": project_path, + "stack_group_path": "A/B", + "keyA": "A", + "keyB": "B", + "shared": "B", + "parent": "A", + } + + config_c = reader.read("A/B/C/config.yaml") + + assert config_c == { + "project_path": project_path, + "stack_group_path": "A/B/C", + "keyA": "A", + "keyB": "B", + "keyC": "C", + "shared": "C", + "parent": "B", + } + def test_read_reads_config_file_with_base_config(self): with self.runner.isolated_filesystem(): - project_path = os.path.abspath('./example') + project_path = os.path.abspath("./example") config_dir = os.path.join(project_path, "config") stack_group_dir = os.path.join(config_dir, "A") os.makedirs(stack_group_dir) config = {"config": "config"} - with open(os.path.join(stack_group_dir, "stack.yaml"), 'w') as\ - config_file: - yaml.safe_dump( - config, stream=config_file, default_flow_style=False - ) - - base_config = { - "base_config": "base_config" - } + with open(os.path.join(stack_group_dir, "stack.yaml"), "w") as config_file: + yaml.safe_dump(config, stream=config_file, default_flow_style=False) + + base_config = {"base_config": "base_config"} self.context.project_path = project_path - config = ConfigReader(self.context).read( - "A/stack.yaml", base_config - ) + config = ConfigReader(self.context).read("A/stack.yaml", base_config) assert config == { "project_path": project_path, "stack_group_path": "A", "config": "config", - "base_config": "base_config" + "base_config": "base_config", } def test_read_with_nonexistant_filepath(self): @@ -132,12 +190,10 @@ def test_read_with_nonexistant_filepath(self): def test_read_with_empty_config_file(self): config_reader = ConfigReader(self.context) - config = config_reader.read( - "account/stack-group/region/subnets.yaml" - ) + config = config_reader.read("account/stack-group/region/subnets.yaml") assert config == { "project_path": self.test_project_path, - "stack_group_path": "account/stack-group/region" + "stack_group_path": "account/stack-group/region", } def test_read_with_templated_config_file(self): @@ -148,15 +204,13 @@ def test_read_with_templated_config_file(self): "region": "region_region", "project_code": "account_project_code", "required_version": "'>1.0'", - "template_bucket_name": "stack_group_template_bucket_name" + "template_bucket_name": "stack_group_template_bucket_name", } os.environ["TEST_ENV_VAR"] = "environment_variable_value" - config = config_reader.read( - "account/stack-group/region/security_groups.yaml" - ) + config = config_reader.read("account/stack-group/region/security_groups.yaml") assert config == { - 'project_path': self.context.project_path, + "project_path": self.context.project_path, "stack_group_path": "account/stack-group/region", "parameters": { "param1": "user_variable_value", @@ -164,59 +218,51 @@ def test_read_with_templated_config_file(self): "param3": "region_region", "param4": "account_project_code", "param5": ">1.0", - "param6": "stack_group_template_bucket_name" - } + "param6": "stack_group_template_bucket_name", + }, } def test_aborts_on_incompatible_version_requirement(self): - config = { - 'required_version': '<0' - } + config = {"required_version": "<0"} with pytest.raises(VersionIncompatibleError): ConfigReader(self.context)._check_version(config) @freeze_time("2012-01-01") - @pytest.mark.parametrize("stack_name,config,expected", [ - ( - "name", - { - "template_bucket_name": "bucket-name", - "template_key_prefix": "prefix", - "region": "eu-west-1" - }, - { - "bucket_name": "bucket-name", - "bucket_key": "prefix/name/2012-01-01-00-00-00-000000Z.json", - "bucket_region": "eu-west-1", - } - ), - ( - "name", - { - "template_bucket_name": "bucket-name", - "region": "eu-west-1" - }, - { - "bucket_name": "bucket-name", - "bucket_key": "name/2012-01-01-00-00-00-000000Z.json", - "bucket_region": "eu-west-1", - } - ), - ( - "name", - { - "template_bucket_name": "bucket-name", - }, - { - "bucket_name": "bucket-name", - "bucket_key": "name/2012-01-01-00-00-00-000000Z.json", - "bucket_region": None, - } - ), - ( - "name", {}, None - ) - ] + @pytest.mark.parametrize( + "stack_name,config,expected", + [ + ( + "name", + { + "template_bucket_name": "bucket-name", + "template_key_prefix": "prefix", + "region": "eu-west-1", + }, + { + "bucket_name": "bucket-name", + "bucket_key": "prefix/name/2012-01-01-00-00-00-000000Z.json", + }, + ), + ( + "name", + {"template_bucket_name": "bucket-name", "region": "eu-west-1"}, + { + "bucket_name": "bucket-name", + "bucket_key": "name/2012-01-01-00-00-00-000000Z.json", + }, + ), + ( + "name", + { + "template_bucket_name": "bucket-name", + }, + { + "bucket_name": "bucket-name", + "bucket_key": "name/2012-01-01-00-00-00-000000Z.json", + }, + ), + ("name", {}, None), + ], ) def test_collect_s3_details(self, stack_name, config, expected): details = ConfigReader._collect_s3_details(stack_name, config) @@ -237,9 +283,8 @@ def test_construct_stacks_constructs_stack( mock_Stack.assert_any_call( name="account/stack-group/region/vpc", project_code="account_project_code", - template_path=os.path.join( - self.context.project_path, "templates/path/to/template" - ), + template_path=None, + template_handler_config={"path": "path/to/template"}, region="region_region", profile="account_profile", parameters={"param1": "val1"}, @@ -248,67 +293,163 @@ def test_construct_stacks_constructs_stack( s3_details=sentinel.s3_details, dependencies=["child/level", "top/level"], iam_role=None, + sceptre_role=None, + iam_role_session_duration=None, + sceptre_role_session_duration=None, role_arn=None, + cloudformation_service_role=None, protected=False, tags={}, external_name=None, notifications=None, on_failure=None, + disable_rollback=False, stack_timeout=0, - required_version='>1.0', - template_bucket_name='stack_group_template_bucket_name', + required_version=">1.0", + template_bucket_name="stack_group_template_bucket_name", template_key_prefix=None, + ignore=False, + obsolete=False, stack_group_config={ - "custom_key": "custom_value" - } + "project_path": self.context.project_path, + "custom_key": "custom_value", + }, ) assert stacks == ({sentinel.stack}, {sentinel.stack}) - @pytest.mark.parametrize("filepaths,expected_stacks", [ - (["A/1.yaml"], {"A/1"}), - (["A/1.yaml", "A/2.yaml", "A/3.yaml"], {"A/3", "A/2", "A/1"}), - (["A/1.yaml", "A/A/1.yaml"], {"A/1", "A/A/1"}), - (["A/1.yaml", "A/A/1.yaml", "A/A/2.yaml"], {"A/1", "A/A/1", "A/A/2"}), - (["A/A/1.yaml", "A/B/1.yaml"], {"A/A/1", "A/B/1"}) - ]) + @pytest.mark.parametrize( + "command_path,filepaths,expected_stacks,expected_command_stacks,full_scan", + [ + ("", ["A/1.yaml"], {"A/1"}, {"A/1"}, False), + ( + "", + ["A/1.yaml", "A/2.yaml", "A/3.yaml"], + {"A/3", "A/2", "A/1"}, + {"A/3", "A/2", "A/1"}, + False, + ), + ("", ["A/1.yaml", "A/A/1.yaml"], {"A/1", "A/A/1"}, {"A/1", "A/A/1"}, False), + ( + "", + ["A/1.yaml", "A/A/1.yaml", "A/A/2.yaml"], + {"A/1", "A/A/1", "A/A/2"}, + {"A/1", "A/A/1", "A/A/2"}, + False, + ), + ( + "", + ["A/A/1.yaml", "A/B/1.yaml"], + {"A/A/1", "A/B/1"}, + {"A/A/1", "A/B/1"}, + False, + ), + ("Abd", ["Abc/1.yaml", "Abd/1.yaml"], {"Abd/1"}, {"Abd/1"}, False), + ( + "Abd", + ["Abc/1.yaml", "Abd/Abc/1.yaml", "Abd/2.yaml"], + {"Abd/2", "Abd/Abc/1"}, + {"Abd/2", "Abd/Abc/1"}, + False, + ), + ( + "Abd/Abc", + ["Abc/1.yaml", "Abd/Abc/1.yaml", "Abd/2.yaml"], + {"Abd/Abc/1"}, + {"Abd/Abc/1"}, + False, + ), + ("Ab", ["Abc/1.yaml", "Abd/1.yaml"], set(), set(), False), + ( + "Abd/Abc", + ["Abc/1.yaml", "Abd/Abc/1.yaml", "Abd/2.yaml"], + {"Abc/1", "Abd/Abc/1", "Abd/2"}, + {"Abd/Abc/1"}, + True, + ), + ], + ) def test_construct_stacks_with_valid_config( - self, filepaths, expected_stacks + self, + command_path, + filepaths, + expected_stacks, + expected_command_stacks, + full_scan, ): project_path, config_dir = self.create_project() for rel_path in filepaths: - config = { "region": "region", "project_code": "project_code", - "template_path": rel_path + "template": { + "path": rel_path, + }, } abs_path = os.path.join(config_dir, rel_path) self.write_config(abs_path, config) self.context.project_path = project_path + self.context.command_path = command_path + self.context.full_scan = full_scan config_reader = ConfigReader(self.context) all_stacks, command_stacks = config_reader.construct_stacks() assert {str(stack) for stack in all_stacks} == expected_stacks + assert {str(stack) for stack in command_stacks} == expected_command_stacks - @pytest.mark.parametrize("filepaths, del_key", [ - (["A/1.yaml"], "project_code"), - (["A/1.yaml"], "region"), - (["A/1.yaml"], "template_path"), - ]) - def test_missing_attr( - self, filepaths, del_key - ): + def test_construct_stacks_with_disable_rollback_command_param(self): project_path, config_dir = self.create_project() - for rel_path in filepaths: + rel_path = "A/1.yaml" + config = { + "region": "region", + "project_code": "project_code", + "template": {"path": rel_path}, + } + abs_path = os.path.join(config_dir, rel_path) + self.write_config(abs_path, config) + self.context.project_path = project_path + self.context.command_params["disable_rollback"] = True + config_reader = ConfigReader(self.context) + all_stacks, command_stacks = config_reader.construct_stacks() + assert list(all_stacks)[0].disable_rollback + + def test_construct_stacks_with_disable_rollback_in_stack_config(self): + project_path, config_dir = self.create_project() + + rel_path = "A/1.yaml" + config = { + "region": "region", + "project_code": "project_code", + "template": {"path": rel_path}, + "disable_rollback": True, + } + + abs_path = os.path.join(config_dir, rel_path) + self.write_config(abs_path, config) + self.context.project_path = project_path + config_reader = ConfigReader(self.context) + all_stacks, command_stacks = config_reader.construct_stacks() + assert list(all_stacks)[0].disable_rollback + + @pytest.mark.parametrize( + "filepaths, del_key", + [ + (["A/1.yaml"], "project_code"), + (["A/1.yaml"], "region"), + ], + ) + def test_missing_attr(self, filepaths, del_key): + project_path, config_dir = self.create_project() + + for rel_path in filepaths: config = { "project_code": "project_code", "region": "region", - "template_path": rel_path + "template_path": rel_path, } # Delete the mandatory key to be tested. del config[del_key] @@ -328,13 +469,14 @@ def test_missing_attr( else: assert False - @pytest.mark.parametrize("filepaths, dependency", [ - (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "A/1.yaml"), - (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "B/1.yaml"), - ]) - def test_existing_dependency( - self, filepaths, dependency - ): + @pytest.mark.parametrize( + "filepaths, dependency", + [ + (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "A/1.yaml"), + (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "B/1.yaml"), + ], + ) + def test_existing_dependency(self, filepaths, dependency): project_path, config_dir = self.create_project() for rel_path in filepaths: @@ -342,8 +484,8 @@ def test_existing_dependency( config = { "project_code": "project_code", "region": "region", - "template_path": rel_path, - "dependencies": [dependency] + "template": {"path": rel_path}, + "dependencies": [dependency], } abs_path = os.path.join(config_dir, rel_path) @@ -358,13 +500,14 @@ def test_existing_dependency( else: assert True - @pytest.mark.parametrize("filepaths, dependency", [ - (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "A/2.yaml"), - (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "1.yaml"), - ]) - def test_missing_dependency( - self, filepaths, dependency - ): + @pytest.mark.parametrize( + "filepaths, dependency", + [ + (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "A/2.yaml"), + (["A/1.yaml", "B/1.yaml", "B/2.yaml"], "1.yaml"), + ], + ) + def test_missing_dependency(self, filepaths, dependency): project_path, config_dir = self.create_project() for rel_path in filepaths: @@ -372,8 +515,8 @@ def test_missing_dependency( config = { "project_code": "project_code", "region": "region", - "template_path": rel_path, - "dependencies": [dependency] + "template": {"path": rel_path}, + "dependencies": [dependency], } abs_path = os.path.join(config_dir, rel_path) @@ -390,3 +533,50 @@ def test_missing_dependency( raise else: assert False + + @pytest.mark.parametrize( + "filepaths, dependency, parent_config_path", + [ + (["A/1.yaml", "A/2.yaml", "B/1.yaml"], "B/1.yaml", "A/config.yaml"), + (["A/1.yaml", "A/2.yaml"], "A/1.yaml", "A/config.yaml"), + ], + ) + def test_inherited_dependency_already_resolved( + self, filepaths, dependency, parent_config_path + ): + project_path, config_dir = self.create_project() + parent_config = {"dependencies": [dependency]} + abs_path = os.path.join(config_dir, parent_config_path) + self.write_config(abs_path, parent_config) + + for rel_path in filepaths: + # Set up config with reference to an existing stack + config = { + "project_code": "project_code", + "region": "region", + "template": {"path": rel_path}, + } + + abs_path = os.path.join(config_dir, rel_path) + self.write_config(abs_path, config) + self.context.project_path = project_path + try: + config_reader = ConfigReader(self.context) + all_stacks, command_stacks = config_reader.construct_stacks() + except Exception: + raise + else: + assert True + + def test_resolve_node_tag(self): + mock_loader = MagicMock(yaml.Loader) + mock_loader.resolve.return_value = "new_tag" + + mock_node = MagicMock(yaml.Node) + mock_node.tag = "old_tag" + mock_node.value = "String" + + config_reader = ConfigReader(self.context) + new_node = config_reader.resolve_node_tag(mock_loader, mock_node) + + assert new_node.tag == "new_tag" diff --git a/tests/test_connection_manager.py b/tests/test_connection_manager.py index 4fab62f89..71bc21621 100644 --- a/tests/test_connection_manager.py +++ b/tests/test_connection_manager.py @@ -1,38 +1,48 @@ # -*- coding: utf-8 -*- +import warnings +from collections import defaultdict +from typing import Union +from unittest.mock import Mock, patch, sentinel, create_autospec -import os +import deprecation import pytest -from mock import Mock, MagicMock, patch, sentinel, ANY -from moto import mock_s3 - from boto3.session import Session -from botocore.exceptions import ClientError, UnknownServiceError +from botocore.exceptions import ClientError +from moto import mock_s3 -from sceptre.connection_manager import ConnectionManager, _retry_boto_call +from sceptre.connection_manager import ( + ConnectionManager, + _retry_boto_call, +) from sceptre.exceptions import RetryLimitExceededError, InvalidAWSCredentialsError class TestConnectionManager(object): - def setup_method(self, test_method): self.stack_name = None self.profile = None - self.iam_role = None + self.sceptre_role = None + self.sceptre_role_session_duration = 3600 self.region = "eu-west-1" + self.environment_variables = { + "AWS_ACCESS_KEY_ID": "sceptre_test_key_id", + "AWS_SECRET_ACCESS_KEY": "sceptre_test_access_key", + } + self.session_class = create_autospec(Session) + self.mock_session: Union[Mock, Session] = self.session_class.return_value + ConnectionManager._boto_sessions = {} ConnectionManager._clients = {} ConnectionManager._stack_keys = {} - # Temporary workaround for https://github.com/spulec/moto/issues/1924 - os.environ.setdefault("AWS_ACCESS_KEY_ID", "sceptre_test_key_id") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "sceptre_test_access_key") - self.connection_manager = ConnectionManager( region=self.region, stack_name=self.stack_name, profile=self.profile, - iam_role=self.iam_role + sceptre_role=self.sceptre_role, + session_class=self.session_class, + get_envs_func=lambda: self.environment_variables, ) def test_connection_manager_initialised_with_no_optional_parameters(self): @@ -50,28 +60,45 @@ def test_connection_manager_initialised_with_all_parameters(self): region=self.region, stack_name="stack", profile="profile", - iam_role="iam_role" + sceptre_role="sceptre_role", + sceptre_role_session_duration=21600, ) assert connection_manager.stack_name == "stack" assert connection_manager.profile == "profile" - assert connection_manager.iam_role == "iam_role" + assert connection_manager.sceptre_role == "sceptre_role" + assert connection_manager.sceptre_role_session_duration == 21600 assert connection_manager.region == self.region assert connection_manager._boto_sessions == {} assert connection_manager._clients == {} assert connection_manager._stack_keys == { - "stack": (self.region, "profile", "iam_role") + "stack": (self.region, "profile", "sceptre_role") } def test_repr(self): self.connection_manager.stack_name = "stack" self.connection_manager.profile = "profile" self.connection_manager.region = "region" - self.connection_manager.iam_role = "iam_role" + self.connection_manager.sceptre_role = "sceptre_role" response = self.connection_manager.__repr__() - assert response == "sceptre.connection_manager.ConnectionManager(" \ - "region='region', profile='profile', stack_name='stack', "\ - "iam_role='iam_role')" + assert ( + response == "sceptre.connection_manager.ConnectionManager(" + "region='region', profile='profile', stack_name='stack', " + "sceptre_role='sceptre_role', sceptre_role_session_duration='None')" + ) + + def test_repr_with_sceptre_role_session_duration(self): + self.connection_manager.stack_name = "stack" + self.connection_manager.profile = "profile" + self.connection_manager.region = "region" + self.connection_manager.sceptre_role = "sceptre_role" + self.connection_manager.sceptre_role_session_duration = 21600 + response = self.connection_manager.__repr__() + assert ( + response == "sceptre.connection_manager.ConnectionManager(" + "region='region', profile='profile', stack_name='stack', " + "sceptre_role='sceptre_role', sceptre_role_session_duration='21600')" + ) def test_boto_session_with_cache(self): self.connection_manager._boto_sessions["test"] = sentinel.boto_session @@ -79,215 +106,724 @@ def test_boto_session_with_cache(self): boto_session = self.connection_manager._boto_sessions["test"] assert boto_session == sentinel.boto_session - @patch("sceptre.connection_manager.boto3.session.Session") - def test_boto_session_with_no_profile( - self, mock_Session - ): - self.connection_manager._boto_sessions = {} + def test__get_session__no_args__no_defaults__makes_boto_session_with_defaults(self): self.connection_manager.profile = None + self.connection_manager.sceptre_role = None - boto_session = self.connection_manager._get_session( - self.connection_manager.profile, self.region, self.iam_role - ) + boto_session = self.connection_manager.get_session() - assert boto_session.isinstance(mock_Session) - mock_Session.assert_called_once_with( + self.session_class.assert_called_once_with( profile_name=None, - region_name="eu-west-1", - aws_access_key_id=ANY, - aws_secret_access_key=ANY, - aws_session_token=ANY + region_name=self.region, + aws_access_key_id=self.environment_variables["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=self.environment_variables["AWS_SECRET_ACCESS_KEY"], + aws_session_token=None, ) + assert boto_session == self.mock_session - @patch("sceptre.connection_manager.boto3.session.Session") - def test_boto_session_with_profile(self, mock_Session): - self.connection_manager._boto_sessions = {} - self.connection_manager.profile = "profile" + def test_get_session__no_args__connection_manager_has_profile__uses_profile(self): + self.connection_manager.profile = "fancy" + self.connection_manager.sceptre_role = None - boto_session = self.connection_manager._get_session( - self.connection_manager.profile, self.region, self.iam_role - ) + boto_session = self.connection_manager.get_session() - assert boto_session.isinstance(mock_Session) - mock_Session.assert_called_once_with( - profile_name="profile", - region_name="eu-west-1", - aws_access_key_id=ANY, - aws_secret_access_key=ANY, - aws_session_token=ANY + self.session_class.assert_called_once_with( + profile_name="fancy", + region_name=self.region, + aws_access_key_id=self.environment_variables["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=self.environment_variables["AWS_SECRET_ACCESS_KEY"], + aws_session_token=None, ) + assert boto_session == self.mock_session - @patch("sceptre.connection_manager.boto3.session.Session") - def test_boto_session_with_no_iam_role( - self, mock_Session + def test_get_session___profile_specified__makes_boto_session_with_passed_profile( + self, ): - self.connection_manager._boto_sessions = {} - self.connection_manager.iam_role = None + self.connection_manager.profile = None - boto_session = self.connection_manager._get_session( - self.profile, self.region, self.connection_manager.iam_role + boto_session = self.connection_manager.get_session(profile="fancy") + + self.session_class.assert_called_once_with( + profile_name="fancy", + region_name=self.region, + aws_access_key_id=self.environment_variables["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=self.environment_variables["AWS_SECRET_ACCESS_KEY"], + aws_session_token=None, ) + assert boto_session == self.mock_session + + def test_get_session__none_for_profile_passed__connection_manager_has_default_profile__uses_no_profile( + self, + ): + self.connection_manager.profile = "default profile" - assert boto_session.isinstance(mock_Session) - mock_Session.assert_called_once_with( + boto_session = self.connection_manager.get_session(profile=None) + + self.session_class.assert_called_once_with( profile_name=None, - region_name="eu-west-1", - aws_access_key_id=ANY, - aws_secret_access_key=ANY, - aws_session_token=ANY + region_name=self.region, + aws_access_key_id=self.environment_variables["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=self.environment_variables["AWS_SECRET_ACCESS_KEY"], + aws_session_token=None, ) + assert boto_session == self.mock_session - boto_session.client().assume_role.assert_not_called() + def test_get_session__no_sceptre_role_passed__no_sceptre_role_on_connection_manager__does_not_assume_role( + self, + ): + self.connection_manager.sceptre_role = None - @patch("sceptre.connection_manager.boto3.session.Session") - def test_boto_session_with_iam_role(self, mock_Session): - self.connection_manager._boto_sessions = {} - self.connection_manager.iam_role = "iam_role" + self.connection_manager.get_session() + self.mock_session.client.assert_not_called() - boto_session = self.connection_manager._get_session( - self.profile, self.region, self.connection_manager.iam_role + def test_get_session__none_passed_for_sceptre_role__sceptre_role_on_connection_manager__does_not_assume_role( + self, + ): + self.connection_manager.sceptre_role = ( + "arn:aws:iam::123456:role/my-path/other-role" ) + self.connection_manager.get_session(sceptre_role=None) - assert boto_session.isinstance(mock_Session) - mock_Session.assert_any_call( - profile_name=None, - region_name="eu-west-1", - aws_access_key_id=ANY, - aws_secret_access_key=ANY, - aws_session_token=ANY - ) + self.mock_session.client.assert_not_called() - boto_session.client().assume_role.assert_called_once_with( - RoleArn=self.connection_manager.iam_role, - RoleSessionName="{0}-session".format( - self.connection_manager.iam_role.split("/")[-1] - ) + @pytest.mark.parametrize( + "connection_manager,arg", + [ + pytest.param( + "arn:aws:iam::123456:role/my-path/my-role", + ConnectionManager.STACK_DEFAULT, + id="role on connection manager", + ), + pytest.param( + "arn:aws:iam::123456:role/my-path/other-role", + "arn:aws:iam::123456:role/my-path/my-role", + id="overrides connection manager", + ), + ], + ) + def test_get_session__sceptre_role__assumes_that_role( + self, connection_manager, arg + ): + self.connection_manager.sceptre_role = connection_manager + + kwargs = {} + if arg != self.connection_manager.STACK_DEFAULT: + kwargs["sceptre_role"] = arg + + self.connection_manager.get_session(**kwargs) + + self.mock_session.client.assert_called_once_with("sts") + expected_role = ( + arg if arg != self.connection_manager.STACK_DEFAULT else connection_manager + ) + self.mock_session.client.return_value.assume_role.assert_called_once_with( + RoleArn=expected_role, RoleSessionName="my-role-session" ) - credentials = boto_session.client().assume_role()["Credentials"] + credentials = self.mock_session.client.return_value.assume_role()["Credentials"] - mock_Session.assert_any_call( - region_name="eu-west-1", + self.session_class.assert_any_call( + region_name=self.region, aws_access_key_id=credentials["AccessKeyId"], aws_secret_access_key=credentials["SecretAccessKey"], - aws_session_token=credentials["SessionToken"] + aws_session_token=credentials["SessionToken"], + ) + + def test_get_session__sceptre_role_and_session_duration_on_connection_manager__uses_session_duration( + self, + ): + self.connection_manager.sceptre_role = "sceptre_role" + self.connection_manager.sceptre_role_session_duration = 21600 + + self.connection_manager.get_session() + + self.mock_session.client.return_value.assume_role.assert_called_once_with( + RoleArn=self.connection_manager.sceptre_role, + RoleSessionName="{0}-session".format( + self.connection_manager.sceptre_role.split("/")[-1] + ), + DurationSeconds=21600, ) - @patch("sceptre.connection_manager.boto3.session.Session") - def test_boto_session_with_iam_role_returning_empty_credentials(self, mock_Session): + def test_get_session__with_sceptre_role__returning_empty_credentials__raises_invalid_aws_credentials_error( + self, + ): self.connection_manager._boto_sessions = {} - self.connection_manager.iam_role = "iam_role" + self.connection_manager.sceptre_role = "sceptre_role" - mock_Session.return_value.get_credentials.side_effect = [ - MagicMock(), None, MagicMock(), MagicMock(), MagicMock() - ] + self.mock_session.get_credentials.return_value = None with pytest.raises(InvalidAWSCredentialsError): - self.connection_manager._get_session( - self.profile, self.region, self.connection_manager.iam_role + self.connection_manager.get_session( + self.profile, self.region, self.connection_manager.sceptre_role ) - @patch("sceptre.connection_manager.boto3.session.Session") - def test_two_boto_sessions(self, mock_Session): - self.connection_manager._boto_sessions = { - "one": mock_Session, - "two": mock_Session - } - - boto_session_1 = self.connection_manager._boto_sessions["one"] - boto_session_2 = self.connection_manager._boto_sessions["two"] - assert boto_session_1 == boto_session_2 - - @patch("sceptre.connection_manager.boto3.session.Session.get_credentials") - def test_get_client_with_no_pre_existing_clients( - self, mock_get_credentials - ): + def test_get_client_with_no_pre_existing_clients(self): service = "s3" region = "eu-west-1" profile = None - iam_role = None + sceptre_role = None stack = self.stack_name client = self.connection_manager._get_client( - service, region, profile, stack, iam_role + service, region, profile, stack, sceptre_role ) - expected_client = Session().client(service) - assert str(type(client)) == str(type(expected_client)) - - @patch("sceptre.connection_manager.boto3.session.Session.get_credentials") - def test_get_client_with_invalid_client_type(self, mock_get_credentials): - service = "invalid_type" - region = "eu-west-1" - iam_role = None - profile = None - stack = self.stack_name - - with pytest.raises(UnknownServiceError): - self.connection_manager._get_client( - service, region, profile, stack, iam_role - ) + expected_client = self.mock_session.client.return_value + assert client == expected_client + self.mock_session.client.assert_any_call(service) - @patch("sceptre.connection_manager.boto3.session.Session.get_credentials") - def test_get_client_with_exisiting_client(self, mock_get_credentials): + def test_get_client_with_existing_client(self): service = "cloudformation" region = "eu-west-1" - iam_role = None + sceptre_role = None profile = None stack = self.stack_name client_1 = self.connection_manager._get_client( - service, region, profile, stack, iam_role + service, region, profile, stack, sceptre_role ) client_2 = self.connection_manager._get_client( - service, region, profile, stack, iam_role + service, region, profile, stack, sceptre_role ) assert client_1 == client_2 + assert self.mock_session.client.call_count == 1 @patch("sceptre.connection_manager.boto3.session.Session.get_credentials") - def test_get_client_with_exisiting_client_and_profile_none( - self, mock_get_credentials + def test_get_client_with_existing_client_and_profile_none( + self, mock_get_credentials ): service = "cloudformation" region = "eu-west-1" - iam_role = None + sceptre_role = None profile = None stack = self.stack_name self.connection_manager.profile = None client_1 = self.connection_manager._get_client( - service, region, profile, stack, iam_role + service, region, profile, stack, sceptre_role ) client_2 = self.connection_manager._get_client( - service, region, profile, stack, iam_role + service, region, profile, stack, sceptre_role ) assert client_1 == client_2 @mock_s3 def test_call_with_valid_service_and_call(self): - service = 's3' - command = 'list_buckets' + service = "s3" + command = "list_buckets" - return_value = self.connection_manager.call(service, command, {}) - assert return_value['ResponseMetadata']['HTTPStatusCode'] == 200 + connection_manager = ConnectionManager(region=self.region) + return_value = connection_manager.call(service, command, {}) + assert return_value["ResponseMetadata"]["HTTPStatusCode"] == 200 @mock_s3 def test_call_with_valid_service_and_stack_name_call(self): - service = 's3' - command = 'list_buckets' + service = "s3" + command = "list_buckets" - connection_manager = ConnectionManager( - region=self.region, - stack_name='stack' + connection_manager = ConnectionManager(region=self.region, stack_name="stack") + + return_value = connection_manager.call(service, command, {}, stack_name="stack") + assert return_value["ResponseMetadata"]["HTTPStatusCode"] == 200 + + def test_call__profile_region_and_role_are_stack_default__uses_instance_settings( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = None + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + expected_client = self.set_up_expected_client( + service, stack_name, instance_profile, self.region, instance_role + ) + + self.connection_manager.call(service, command) + expected_client.list_buckets.assert_any_call() + + def set_connection_manager_vars(self, profile, region, sceptre_role): + self.connection_manager.region = region + self.connection_manager.profile = profile + self.connection_manager.sceptre_role = sceptre_role + + def set_up_expected_client( + self, service, stack_name, profile, region, sceptre_role + ): + self.connection_manager._clients = clients = defaultdict(Mock) + clients[(service, region, profile, stack_name, sceptre_role)] = expected = Mock( + name="expected" + ) + return expected + + def set_target_stack_settings(self, stack_name, profile, region, role): + self.connection_manager._stack_keys = settings = defaultdict( + lambda: ("wrong", "wrong", "wrong") + ) + settings[stack_name] = (region, profile, role) + + def test_call__profile_region_set__role_is_stack_default__uses_instance_role(self): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = None + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + expected_profile = "new profile" + expected_region = "us-west-800" + expected_client = self.set_up_expected_client( + service, stack_name, expected_profile, expected_region, instance_role + ) + + self.connection_manager.call( + service, command, profile=expected_profile, region=expected_region + ) + expected_client.list_buckets.assert_any_call() + + def test_call__profile_region_set__role_is_none__nullifies_role(self): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = None + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + expected_profile = "new profile" + expected_region = "us-west-800" + expected_role = None + expected_client = self.set_up_expected_client( + service, stack_name, expected_profile, expected_region, expected_role + ) + + self.connection_manager.call( + service, + command, + profile=expected_profile, + region=expected_region, + sceptre_role=expected_role, + ) + expected_client.list_buckets.assert_any_call() + + def test_call__stack_name_set_and_cached__profile_region_and_role_are_stack_default__uses_target_stack_settings( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + target_profile = "new profile" + target_region = "us-west-800" + target_role = "roley role" + self.set_target_stack_settings( + stack_name, target_profile, target_region, target_role + ) + expected_client = self.set_up_expected_client( + service, stack_name, target_profile, target_region, target_role + ) + + self.connection_manager.call( + service, + command, + stack_name=stack_name, + ) + expected_client.list_buckets.assert_any_call() + + def test_call__stack_name_set_not_cached__profile_region_and_role_are_stack_default__uses_target_stack_settings( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + expected_client = self.set_up_expected_client( + service, stack_name, instance_profile, self.region, instance_role ) - return_value = connection_manager.call( - service, command, {}, stack_name='stack' + self.connection_manager.call( + service, + command, + stack_name=stack_name, ) - assert return_value['ResponseMetadata']['HTTPStatusCode'] == 200 + expected_client.list_buckets.assert_any_call() + def test_call__stack_name_set_and_cached__profile_region_and_role_are_none__uses_current_stack_settings( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + target_profile = "new profile" + target_region = "us-west-800" + target_role = "roley role" + self.set_target_stack_settings( + stack_name, target_profile, target_region, target_role + ) + expected_client = self.set_up_expected_client( + service, stack_name, target_profile, target_region, target_role + ) + + self.connection_manager.call( + service, + command, + stack_name=stack_name, + profile=None, + region=None, + sceptre_role=None, + ) + expected_client.list_buckets.assert_any_call() + + def test_call__stack_name_set_not_cached__profile_region_and_role_are_none__uses_current_stack_settings( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + expected_client = self.set_up_expected_client( + service, stack_name, instance_profile, self.region, instance_role + ) -class TestRetry(): + self.connection_manager.call( + service, + command, + stack_name=stack_name, + profile=None, + region=None, + sceptre_role=None, + ) + expected_client.list_buckets.assert_any_call() + + def test_call__stack_name_set_and_cached__profile_and_region_are_stack_default_and_role_is_none__nullifies_role( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + target_profile = "new profile" + target_region = "us-west-800" + target_role = "roley role" + self.set_target_stack_settings( + stack_name, target_profile, target_region, target_role + ) + + expected_client = self.set_up_expected_client( + service, stack_name, target_profile, target_region, None + ) + + self.connection_manager.call( + service, + command, + stack_name=stack_name, + sceptre_role=None, + ) + expected_client.list_buckets.assert_any_call() + + def test_call__stack_name_set_not_cached__profile_and_region_are_stack_default_and_role_is_none__nullifies_role( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + expected_client = self.set_up_expected_client( + service, stack_name, instance_profile, self.region, None + ) + + self.connection_manager.call( + service, + command, + stack_name=stack_name, + sceptre_role=None, + ) + expected_client.list_buckets.assert_any_call() + + def test_call__invoked_with_iam_role_kwarg__emits_deprecation_warning(self): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = None + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + self.set_up_expected_client( + service, stack_name, instance_profile, self.region, instance_role + ) + + with warnings.catch_warnings(record=True) as recorded: + self.connection_manager.call(service, command, iam_role="new role") + + assert len(recorded) == 1 + assert issubclass(recorded[0].category, DeprecationWarning) + + def test_call__stack_name_set_and_cached__invoked_with_iam_role_kwarg__emits_deprecation_warning( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + target_profile = "new profile" + target_region = "us-west-800" + target_role = "roley role" + self.set_target_stack_settings( + stack_name, target_profile, target_region, target_role + ) + self.set_up_expected_client( + service, stack_name, target_profile, target_region, target_role + ) + with warnings.catch_warnings(record=True) as recorded: + self.connection_manager.call( + service, + command, + stack_name=stack_name, + profile=None, + region=None, + iam_role=None, + ) + + assert len(recorded) == 1 + assert issubclass(recorded[0].category, DeprecationWarning) + + def test_call__stack_name_set_not_cached__invoked_with_iam_role_kwarg__emits_deprecation_warning( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + self.set_up_expected_client( + service, stack_name, instance_profile, self.region, instance_role + ) + with warnings.catch_warnings(record=True) as recorded: + self.connection_manager.call( + service, + command, + stack_name=stack_name, + profile=None, + region=None, + iam_role=None, + ) + + assert len(recorded) == 1 + assert issubclass(recorded[0].category, DeprecationWarning) + + def test_call__invoked_with_iam_role__uses_that_as_sceptre_role(self): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = None + expected_role = "new role" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + expected_client = self.set_up_expected_client( + service, stack_name, instance_profile, self.region, expected_role + ) + with warnings.catch_warnings(): + self.connection_manager.call(service, command, iam_role=expected_role) + + expected_client.list_buckets.assert_any_call() + + def test_call__stack_name_set_and_cached__invoked_with_iam_role__uses_that_as_sceptre_role( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + target_profile = "new profile" + target_region = "us-west-800" + target_role = "roley role" + self.set_target_stack_settings( + stack_name, target_profile, target_region, target_role + ) + + expected_role = "new role" + expected_client = self.set_up_expected_client( + service, stack_name, target_profile, target_region, expected_role + ) + + with warnings.catch_warnings(): + self.connection_manager.call( + service, + command, + stack_name=stack_name, + iam_role=expected_role, + ) + expected_client.list_buckets.assert_any_call() + + def test_call__stack_name_set_not_cached__invoked_with_iam_role__uses_that_as_sceptre_role( + self, + ): + service = "s3" + command = "list_buckets" + instance_profile = "profile" + instance_role = "role" + stack_name = "target" + self.set_connection_manager_vars(instance_profile, self.region, instance_role) + + expected_role = "new role" + expected_client = self.set_up_expected_client( + service, stack_name, instance_profile, self.region, expected_role + ) + + with warnings.catch_warnings(): + self.connection_manager.call( + service, + command, + stack_name=stack_name, + iam_role=expected_role, + ) + expected_client.list_buckets.assert_any_call() + + def test_create_session_environment_variables__no_token__returns_envs_dict(self): + self.mock_session.configure_mock( + **{ + "region_name": "us-west-2", + "get_credentials.return_value.access_key": "new_access_key", + "get_credentials.return_value.secret_key": "new_secret_key", + "get_credentials.return_value.token": None, + } + ) + + result = self.connection_manager.create_session_environment_variables() + expected = { + "AWS_ACCESS_KEY_ID": "new_access_key", + "AWS_SECRET_ACCESS_KEY": "new_secret_key", + "AWS_DEFAULT_REGION": "us-west-2", + "AWS_REGION": "us-west-2", + } + assert expected == result + + def test_create_session_environment_variables__has_session_token__returns_envs_dict_with_token( + self, + ): + self.mock_session.configure_mock( + **{ + "region_name": "us-west-2", + "get_credentials.return_value.access_key": "new_access_key", + "get_credentials.return_value.secret_key": "new_secret_key", + "get_credentials.return_value.token": "my token", + } + ) + + result = self.connection_manager.create_session_environment_variables() + expected = { + "AWS_ACCESS_KEY_ID": "new_access_key", + "AWS_SECRET_ACCESS_KEY": "new_secret_key", + "AWS_DEFAULT_REGION": "us-west-2", + "AWS_REGION": "us-west-2", + "AWS_SESSION_TOKEN": "my token", + } + assert expected == result + + def test_create_session_environment_variables__include_system_envs_true__adds_envs_removing_profile_and_token( + self, + ): + self.environment_variables.update( + AWS_PROFILE="my_profile", # We expect this popped out + AWS_SESSION_TOKEN="my token", # This should be removed if there's no token + OTHER="value-blah-blah", # we expect this to be in dictionary coming out, + ) + + self.mock_session.configure_mock( + **{ + "region_name": "us-west-2", + "get_credentials.return_value.access_key": "new_access_key", + "get_credentials.return_value.secret_key": "new_secret_key", + "get_credentials.return_value.token": None, + } + ) + + result = self.connection_manager.create_session_environment_variables( + include_system_envs=True + ) + expected = { + "AWS_ACCESS_KEY_ID": "new_access_key", + "AWS_SECRET_ACCESS_KEY": "new_secret_key", + "AWS_DEFAULT_REGION": "us-west-2", + "AWS_REGION": "us-west-2", + "OTHER": "value-blah-blah", + } + assert expected == result + + def test_create_session_environment_variables__include_system_envs_false__does_not_add_system_envs( + self, + ): + self.environment_variables.update( + AWS_PROFILE="my_profile", # We expect this popped out + AWS_SESSION_TOKEN="my token", # This should be removed if there's no token + OTHER="value-blah-blah", # we expect this to be in dictionary coming out, + ) + + self.mock_session.configure_mock( + **{ + "region_name": "us-west-2", + "get_credentials.return_value.access_key": "new_access_key", + "get_credentials.return_value.secret_key": "new_secret_key", + "get_credentials.return_value.token": None, + } + ) + + result = self.connection_manager.create_session_environment_variables( + include_system_envs=False + ) + expected = { + "AWS_ACCESS_KEY_ID": "new_access_key", + "AWS_SECRET_ACCESS_KEY": "new_secret_key", + "AWS_DEFAULT_REGION": "us-west-2", + "AWS_REGION": "us-west-2", + } + assert expected == result + + @deprecation.fail_if_not_removed + def test_iam_role__is_removed_on_removal_version(self): + self.connection_manager.iam_role + + @deprecation.fail_if_not_removed + def test_iam_role_session_duration__is_removed_on_removal_version(self): + self.connection_manager.iam_role_session_duration + + def test_init__iam_role_fields_resolve_to_sceptre_role_fields(self): + connection_manager = ConnectionManager( + region="us-west-2", + sceptre_role="sceptre_role", + sceptre_role_session_duration=123456, + ) + assert connection_manager.iam_role == "sceptre_role" + assert connection_manager.iam_role_session_duration == 123456 + + +class TestRetry: def test_retry_boto_call_returns_response_correctly(self): def func(*args, **kwargs): return sentinel.response @@ -297,21 +833,14 @@ def func(*args, **kwargs): assert response == sentinel.response @patch("sceptre.connection_manager.time.sleep") - def test_retry_boto_call_pauses_when_request_limit_hit( - self, mock_sleep - ): + def test_retry_boto_call_pauses_when_request_limit_hit(self, mock_sleep): mock_func = Mock() mock_func.side_effect = [ ClientError( - { - "Error": { - "Code": "Throttling", - "Message": "Request limit hit" - } - }, - sentinel.operation + {"Error": {"Code": "Throttling", "Message": "Request limit hit"}}, + sentinel.operation, ), - sentinel.response + sentinel.response, ] # The attribute function.__name__ is required by the decorator @wraps. mock_func.__name__ = "mock_func" @@ -322,13 +851,7 @@ def test_retry_boto_call_pauses_when_request_limit_hit( def test_retry_boto_call_raises_non_throttling_error(self): mock_func = Mock() mock_func.side_effect = ClientError( - { - "Error": { - "Code": 500, - "Message": "Boom!" - } - }, - sentinel.operation + {"Error": {"Code": 500, "Message": "Boom!"}}, sentinel.operation ) # The attribute function.__name__ is required by the decorator @wraps. mock_func.__name__ = "mock_func" @@ -339,18 +862,11 @@ def test_retry_boto_call_raises_non_throttling_error(self): assert e.value.response["Error"]["Message"] == "Boom!" @patch("sceptre.connection_manager.time.sleep") - def test_retry_boto_call_raises_retry_limit_exceeded_exception( - self, mock_sleep - ): + def test_retry_boto_call_raises_retry_limit_exceeded_exception(self, mock_sleep): mock_func = Mock() mock_func.side_effect = ClientError( - { - "Error": { - "Code": "Throttling", - "Message": "Request limit hit" - } - }, - sentinel.operation + {"Error": {"Code": "Throttling", "Message": "Request limit hit"}}, + sentinel.operation, ) # The attribute function.__name__ is required by the decorator @wraps. mock_func.__name__ = "mock_func" diff --git a/tests/test_context.py b/tests/test_context.py index 0aef85245..bb9694aa2 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,10 +1,9 @@ from os import path -from mock import sentinel +from unittest.mock import sentinel from sceptre.context import SceptreContext class TestSceptreContext(object): - def setup_method(self, test_method): self.templates_path = "templates" self.config_path = "config" @@ -14,25 +13,27 @@ def test_context_with_path(self): self.context = SceptreContext( project_path="project_path/to/sceptre", command_path="command-path", + command_params=sentinel.command_params, user_variables=sentinel.user_variables, options=sentinel.options, output_format=sentinel.output_format, no_colour=sentinel.no_colour, - ignore_dependencies=sentinel.ignore_dependencies + ignore_dependencies=sentinel.ignore_dependencies, ) sentinel.project_path = "project_path/to/sceptre" - assert self.context.project_path == sentinel.project_path + assert self.context.project_path.replace(path.sep, "/") == sentinel.project_path def test_full_config_path_returns_correct_path(self): context = SceptreContext( project_path="project_path", command_path="command-path", + command_params=sentinel.command_params, user_variables=sentinel.user_variables, options=sentinel.options, output_format=sentinel.output_format, no_colour=sentinel.no_colour, - ignore_dependencies=sentinel.ignore_dependencies + ignore_dependencies=sentinel.ignore_dependencies, ) full_config_path = path.join("project_path", self.config_path) @@ -42,15 +43,14 @@ def test_full_command_path_returns_correct_path(self): context = SceptreContext( project_path="project_path", command_path="command", + command_params=sentinel.command_params, user_variables=sentinel.user_variables, options=sentinel.options, output_format=sentinel.output_format, no_colour=sentinel.no_colour, - ignore_dependencies=sentinel.ignore_dependencies + ignore_dependencies=sentinel.ignore_dependencies, ) - full_command_path = path.join("project_path", - self.config_path, - "command") + full_command_path = path.join("project_path", self.config_path, "command") assert context.full_command_path() == full_command_path @@ -58,11 +58,35 @@ def test_full_templates_path_returns_correct_path(self): context = SceptreContext( project_path="project_path", command_path="command", + command_params=sentinel.command_params, user_variables=sentinel.user_variables, options=sentinel.options, output_format=sentinel.output_format, no_colour=sentinel.no_colour, - ignore_dependencies=sentinel.ignore_dependencies + ignore_dependencies=sentinel.ignore_dependencies, ) full_templates_path = path.join("project_path", self.templates_path) assert context.full_templates_path() == full_templates_path + + def test_clone__returns_full_clone_of_context(self): + context = SceptreContext( + project_path="project_path", + command_path="command", + command_params={"params": "variables"}, + user_variables={"user": "variables"}, + options={"hello": "there"}, + output_format=sentinel.output_format, + no_colour=sentinel.no_colour, + ignore_dependencies=sentinel.ignore_dependencies, + ) + clone = context.clone() + assert clone is not context + assert clone.project_path == context.project_path + assert clone.command_path == context.command_path + assert clone.user_variables == context.user_variables + assert clone.user_variables is not context.user_variables + assert clone.options == context.options + assert clone.options is not context.options + assert clone.output_format == context.output_format + assert clone.no_colour == context.no_colour + assert clone.ignore_dependencies == context.ignore_dependencies diff --git a/tests/test_diffing/__init__.py b/tests/test_diffing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_diffing/test_diff_writer.py b/tests/test_diffing/test_diff_writer.py new file mode 100644 index 000000000..f2e8e7e82 --- /dev/null +++ b/tests/test_diffing/test_diff_writer.py @@ -0,0 +1,467 @@ +import difflib +from copy import deepcopy +from io import StringIO +from typing import TextIO +from unittest.mock import Mock + +import cfn_flip +import pytest +import yaml +from deepdiff import DeepDiff + +from sceptre.diffing.diff_writer import ( + DiffWriter, + DeepDiffWriter, + deepdiff_json_defaults, + DiffLibWriter, + ColouredDiffLibWriter, +) +from sceptre.diffing.stack_differ import StackDiff, DiffType, StackConfiguration +from colorama import Fore + + +class ImplementedDiffWriter(DiffWriter): + def __init__( + self, + stack_diff: StackDiff, + output_stream: TextIO, + output_format: str, + capturing_mock: Mock, + ): + super().__init__(stack_diff, output_stream, output_format) + self.capturing_mock = capturing_mock + + def dump_diff(self, diff: DiffType) -> str: + return self.capturing_mock.dump_diff(diff) + + @property + def has_config_difference(self) -> bool: + return self.capturing_mock.has_config_difference + + @property + def has_template_difference(self) -> bool: + return self.capturing_mock.has_template_difference + + +class TestDiffWriter: + def setup_method(self, method): + self.diff_output = "diff" + self.capturing_mock = Mock(**{"dump_diff.return_value": self.diff_output}) + self.stack_name = "stack" + self.template_diff = Mock() + self.config_diff = Mock() + self.is_deployed = True + self.generated_template = "my template" + self.output_format = "yaml" + + self.output_stream = StringIO() + + self.diff_detected_message = ( + f"--> Difference detected for stack {self.stack_name}!" + ) + + @property + def generated_config(self): + return StackConfiguration( + stack_name=self.stack_name, + parameters={}, + stack_tags={}, + notifications=[], + cloudformation_service_role=None, + ) + + @property + def diff(self): + return StackDiff( + self.stack_name, + self.template_diff, + self.config_diff, + self.is_deployed, + self.generated_config, + self.generated_template, + ) + + @property + def writer(self): + return ImplementedDiffWriter( + self.diff, self.output_stream, self.output_format, self.capturing_mock + ) + + def assert_expected_output(self, *expected_segments): + expected_segments = [f"{line}\n" for line in expected_segments] + joined = "".join(expected_segments) + expected_split_lines = joined.splitlines() + + received_lines = self.output_stream.getvalue().splitlines() + diff = list( + difflib.unified_diff( + received_lines, + expected_split_lines, + fromfile="actual", + tofile="expected", + ) + ) + assert not diff, "\n".join(diff) + + def test_write__no_difference__writes_no_difference(self): + self.capturing_mock.has_config_difference = False + self.capturing_mock.has_template_difference = False + + self.writer.write() + + self.assert_expected_output( + DiffWriter.STAR_BAR, f"No difference to deployed stack {self.stack_name}" + ) + + @pytest.mark.parametrize( + "output_format, config_serializer", + [ + pytest.param("yaml", cfn_flip.dump_yaml, id="output format is yaml"), + pytest.param("json", cfn_flip.dump_json, id="output format is json"), + pytest.param("text", cfn_flip.dump_yaml, id="output format is text"), + ], + ) + def test_write__new_stack__writes_new_stack_config_and_template( + self, output_format, config_serializer + ): + self.is_deployed = False + self.output_format = output_format + + self.writer.write() + + self.assert_expected_output( + DiffWriter.STAR_BAR, + self.diff_detected_message, + "This stack is not deployed yet!", + DiffWriter.LINE_BAR, + "New Config:", + "", + config_serializer(dict(self.generated_config._asdict())), + DiffWriter.LINE_BAR, + "New Template:", + "", + self.generated_template, + ) + + def test_write__only_config_is_different__writes_config_difference(self): + self.capturing_mock.has_config_difference = True + self.capturing_mock.has_template_difference = False + + self.writer.write() + + self.assert_expected_output( + DiffWriter.STAR_BAR, + self.diff_detected_message, + DiffWriter.LINE_BAR, + f"Config difference for {self.stack_name}:", + "", + self.diff_output, + DiffWriter.LINE_BAR, + "No template difference", + ) + + def test_write__only_template_is_different__writes_template_difference(self): + self.capturing_mock.has_config_difference = False + self.capturing_mock.has_template_difference = True + + self.writer.write() + + self.assert_expected_output( + DiffWriter.STAR_BAR, + self.diff_detected_message, + DiffWriter.LINE_BAR, + "No stack config difference", + DiffWriter.LINE_BAR, + f"Template difference for {self.stack_name}:", + "", + self.diff_output, + ) + + def test_write__config_and_template_are_different__writes_both_differences(self): + self.capturing_mock.has_config_difference = True + self.capturing_mock.has_template_difference = True + + self.writer.write() + + self.assert_expected_output( + DiffWriter.STAR_BAR, + self.diff_detected_message, + DiffWriter.LINE_BAR, + f"Config difference for {self.stack_name}:", + "", + self.diff_output, + DiffWriter.LINE_BAR, + f"Template difference for {self.stack_name}:", + "", + self.diff_output, + ) + + +class TestDeepDiffWriter: + def setup_method(self, method): + self.stack_name = "stack" + + self.is_deployed = True + self.output_format = "yaml" + + self.output_stream = StringIO() + + self.config1 = StackConfiguration( + stack_name=self.stack_name, + parameters={}, + stack_tags={}, + notifications=[], + cloudformation_service_role=None, + ) + + self.config2 = deepcopy(self.config1) + + self.template1 = "template" + self.template2 = "template" + + @property + def template_diff(self): + return DeepDiff(self.template1, self.template2) + + @property + def config_diff(self): + return DeepDiff(self.config1, self.config2) + + @property + def diff(self): + return StackDiff( + self.stack_name, + self.template_diff, + self.config_diff, + self.is_deployed, + self.config1, + self.template1, + ) + + @property + def writer(self): + return DeepDiffWriter( + self.diff, + self.output_stream, + self.output_format, + ) + + def test_has_config_difference__config_difference_is_present__returns_true(self): + self.config2.parameters["new_key"] = "new value" + assert self.writer.has_config_difference + + def test_has_config_difference__config_difference_is_absent__returns_false(self): + assert self.writer.has_config_difference is False + + def test_has_template_difference__template_difference_is_present__returns_true( + self, + ): + self.template2 = "new" + assert self.writer.has_template_difference + + def test_has_template_difference__template_difference_is_absent__returns_false( + self, + ): + assert self.writer.has_template_difference is False + + def test_dump_diff__output_format_is_json__outputs_to_json(self): + self.output_format = "json" + self.config2.parameters["new_key"] = "new value" + + result = self.writer.dump_diff(self.config_diff) + expected = self.config_diff.to_json( + indent=4, default_mapping=deepdiff_json_defaults + ) + assert result == expected + + def test_dump_diff__output_format_is_yaml__outputs_to_yaml(self): + self.output_format = "yaml" + self.config2.parameters["new_key"] = "new value" + + result = self.writer.dump_diff(self.config_diff) + expected_dict = self.config_diff.to_dict() + expected_yaml = yaml.dump(expected_dict, indent=4) + assert result == expected_yaml + + def test_dump_diff__output_format_is_text__outputs_to_yaml(self): + self.output_format = "text" + self.config2.parameters["new_key"] = "new value" + + result = self.writer.dump_diff(self.config_diff) + expected_dict = self.config_diff.to_dict() + expected_yaml = yaml.dump(expected_dict, indent=4) + assert result == expected_yaml + + def test_dump_diff__output_format_is_yaml__diff_has_multiline_strings__strips_out_extra_spaces( + self, + ): + self.config1.parameters["long_param"] = "here \nis \nmy \nlong \nstring" + self.config2.parameters["long_param"] = "here \nis \nmy \nother \nlong \nstring" + + dumped = self.writer.dump_diff(self.config_diff) + loaded = yaml.safe_load(dumped) + assert ( + " " + not in loaded["values_changed"]["root.parameters['long_param']"][ + "new_value" + ] + ) + assert ( + " " + not in loaded["values_changed"]["root.parameters['long_param']"][ + "old_value" + ] + ) + expected_diff = "\n".join( + difflib.unified_diff( + self.config1.parameters["long_param"].splitlines(), + self.config2.parameters["long_param"].splitlines(), + lineterm="", + ) + ).replace(" \n", "\n") + assert ( + expected_diff + == loaded["values_changed"]["root.parameters['long_param']"]["diff"] + ) + + +class TestDiffLibWriter: + def setup_method(self, method): + self.stack_name = "stack" + + self.is_deployed = True + self.output_format = "yaml" + + self.output_stream = StringIO() + + self.config1 = StackConfiguration( + stack_name=self.stack_name, + parameters={}, + stack_tags={}, + notifications=[], + cloudformation_service_role=None, + ) + + self.config2 = deepcopy(self.config1) + + self.template1 = "template" + self.template2 = "template" + + @property + def template_diff(self): + return list(difflib.unified_diff(self.template1, self.template2)) + + @property + def config_diff(self): + config_1 = yaml.dump(dict(self.config1._asdict())).splitlines() + config_2 = yaml.dump(dict(self.config2._asdict())).splitlines() + return list(difflib.unified_diff(config_1, config_2)) + + @property + def diff(self): + return StackDiff( + self.stack_name, + self.template_diff, + self.config_diff, + self.is_deployed, + self.config1, + self.template1, + ) + + @property + def writer(self): + return DiffLibWriter( + self.diff, + self.output_stream, + self.output_format, + ) + + def test_has_config_difference__config_difference_is_present__returns_true(self): + self.config2.parameters["new_key"] = "new value" + assert self.writer.has_config_difference + + def test_has_config_difference__config_difference_is_absent__returns_false(self): + assert self.writer.has_config_difference is False + + def test_has_template_difference__template_difference_is_present__returns_true( + self, + ): + self.template2 = "new" + assert self.writer.has_template_difference + + def test_has_template_difference__template_difference_is_absent__returns_false( + self, + ): + assert self.writer.has_template_difference is False + + def test_dump_diff__returns_joined_list(self): + result = self.writer.dump_diff(self.diff.config_diff) + expected = "\n".join(self.diff.config_diff) + assert result == expected + + +class TestColouredDiffLibWriter: + def setup_method(self, method): + self.stack_name = "stack" + + self.is_deployed = True + self.output_format = "yaml" + + self.output_stream = StringIO() + + self.config1 = StackConfiguration( + stack_name=self.stack_name, + parameters={}, + stack_tags={}, + notifications=[], + cloudformation_service_role=None, + ) + + self.template1 = "foo" + + @property + def template_diff(self): + return [ + "--- file1.txt 2018-01-11 10:39:38.237464052 +0000\n", + "+++ file2.txt 2018-01-11 10:40:00.323423021 +0000\n", + "@@ -1,4 +1,4 @@\n", + " cat\n", + "-mv\n", + "-comm\n", + " cp\n", + "+diff\n", + "+comm\n", + ] + + @property + def config_diff(self): + return [] + + @property + def diff(self): + return StackDiff( + self.stack_name, + self.template_diff, + self.config_diff, + self.is_deployed, + self.config1, + self.template1, + ) + + @property + def writer(self): + return ColouredDiffLibWriter(self.diff, self.output_stream, self.output_format) + + def test_lines_are_coloured(self): + coloured = ( + f"{Fore.RED}--- file1.txt 2018-01-11 10:39:38.237464052 +0000\n{Fore.RESET}\n" + f"{Fore.GREEN}+++ file2.txt 2018-01-11 10:40:00.323423021 +0000\n{Fore.RESET}\n" + "@@ -1,4 +1,4 @@\n\n" + " cat\n\n" + f"{Fore.RED}-mv\n{Fore.RESET}\n" + f"{Fore.RED}-comm\n{Fore.RESET}\n" + " cp\n\n" + f"{Fore.GREEN}+diff\n{Fore.RESET}\n" + f"{Fore.GREEN}+comm\n{Fore.RESET}" + ) + assert self.writer.dump_diff(self.template_diff) == coloured diff --git a/tests/test_diffing/test_stack_differ.py b/tests/test_diffing/test_stack_differ.py new file mode 100644 index 000000000..0329195e2 --- /dev/null +++ b/tests/test_diffing/test_stack_differ.py @@ -0,0 +1,694 @@ +import difflib +import json +from collections import defaultdict +from copy import deepcopy +from typing import Union, Optional +from unittest.mock import Mock, PropertyMock + +import cfn_flip +import pytest +import yaml + +from sceptre.diffing.stack_differ import ( + StackDiffer, + StackConfiguration, + DiffType, + DeepDiffStackDiffer, + DifflibStackDiffer, +) +from sceptre.plan.actions import StackActions +from sceptre.stack import Stack + + +class ImplementedStackDiffer(StackDiffer): + def __init__(self, command_capturer: Mock): + super().__init__() + self.command_capturer = command_capturer + + def compare_templates(self, deployed: str, generated: str) -> DiffType: + return self.command_capturer.compare_templates(deployed, generated) + + def compare_stack_configurations( + self, deployed: Optional[StackConfiguration], generated: StackConfiguration + ) -> DiffType: + return self.command_capturer.compare_stack_configurations(deployed, generated) + + +class TestStackDiffer: + def setup_method(self, method): + self.name = "my/stack" + self.external_name = "full-stack-name" + self.cloudformation_service_role = "cloudformation_service_role" + self.parameters_on_stack_config = {"param": "some_value"} + self.tags = {"tag_name": "tag_value"} + self.notifications = ["notification_arn1"] + self.sceptre_user_data = {} + + self.deployed_parameters = deepcopy(self.parameters_on_stack_config) + self.deployed_parameter_defaults = {} + self.deployed_no_echo_parameters = [] + self.deployed_parameter_types = defaultdict(lambda: "String") + self.local_no_echo_parameters = [] + self.deployed_tags = dict(self.tags) + self.deployed_notification_arns = list(self.notifications) + self.deployed_cloudformation_service_role = self.cloudformation_service_role + + self.command_capturer = Mock() + self.differ = ImplementedStackDiffer(self.command_capturer) + self.stack_status = "CREATE_COMPLETE" + + self._stack = None + self._actions = None + self._parameters = None + + @property + def parameters_on_stack(self): + if self._parameters is None: + self._parameters = deepcopy(self.parameters_on_stack_config) + return self._parameters + + @property + def stack(self) -> Union[Stack, Mock]: + if not self._stack: + self._stack = Mock( + spec=Stack, + external_name=self.external_name, + _parameters=self.parameters_on_stack, + cloudformation_service_role=self.cloudformation_service_role, + tags=self.tags, + notifications=self.notifications, + __sceptre_user_data=self.sceptre_user_data, + ) + self._stack.name = self.name + type(self._stack).parameters = PropertyMock( + side_effect=lambda: self.parameters_on_stack + ) + return self._stack + + @property + def actions(self) -> Union[StackActions, Mock]: + if not self._actions: + self._actions = Mock( + **{ + "spec": StackActions, + "stack": self.stack, + "describe.side_effect": self.describe_stack, + "fetch_remote_template_summary.side_effect": self.get_remote_template_summary, + "fetch_local_template_summary.side_effect": self.get_local_template_summary, + } + ) + return self._actions + + def describe_stack(self): + return { + "Stacks": [ + { + "StackName": self.stack.external_name, + "Parameters": [ + { + "ParameterKey": key, + "ParameterValue": value, + "ResolvedValue": "I'm resolved and don't matter for the diff!", + } + for key, value in self.deployed_parameters.items() + ], + "StackStatus": self.stack_status, + "NotificationARNs": self.deployed_notification_arns, + "RoleARN": self.deployed_cloudformation_service_role, + "Tags": [ + {"Key": key, "Value": value} + for key, value in self.deployed_tags.items() + ], + }, + ], + } + + def get_remote_template_summary(self): + params = [] + for param, value in self.deployed_parameters.items(): + entry = { + "ParameterKey": param, + "ParameterType": self.deployed_parameter_types[param], + } + if param in self.deployed_parameter_defaults: + default_value = self.deployed_parameter_defaults[param] + if "List" in entry["ParameterType"]: + default_value = ", ".join( + val.strip() for val in default_value.split(",") + ) + entry["DefaultValue"] = default_value + if param in self.deployed_no_echo_parameters: + entry["NoEcho"] = True + + params.append(entry) + + return {"Parameters": params} + + def get_local_template_summary(self): + params = [] + for param, value in self.parameters_on_stack.items(): + entry = {"ParameterKey": param} + if param in self.local_no_echo_parameters: + entry["NoEcho"] = True + params.append(entry) + + return {"Parameters": params} + + @property + def expected_generated_config(self): + return StackConfiguration( + stack_name=self.external_name, + parameters=self.parameters_on_stack_config, + stack_tags=deepcopy(self.tags), + notifications=deepcopy(self.notifications), + cloudformation_service_role=self.cloudformation_service_role, + ) + + @property + def expected_deployed_config(self): + return StackConfiguration( + stack_name=self.external_name, + parameters=self.deployed_parameters, + stack_tags=deepcopy(self.deployed_tags), + notifications=deepcopy(self.deployed_notification_arns), + cloudformation_service_role=self.deployed_cloudformation_service_role, + ) + + def test_diff__compares_deployed_template_to_generated_template(self): + self.differ.diff(self.actions) + + self.command_capturer.compare_templates.assert_called_with( + self.actions.fetch_remote_template.return_value, + self.actions.generate.return_value, + ) + + def test_diff__template_diff_is_value_returned_by_implemented_differ(self): + diff = self.differ.diff(self.actions) + + assert ( + diff.template_diff == self.command_capturer.compare_templates.return_value + ) + + def test_diff__compares_deployed_stack_config_to_generated_stack_config(self): + self.deployed_parameters["new"] = "value" + + self.differ.diff(self.actions) + + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_deployed_config, self.expected_generated_config + ) + + def test_diff__config_diff_is_value_returned_by_implemented_differ(self): + diff = self.differ.diff(self.actions) + + assert ( + diff.config_diff + == self.command_capturer.compare_stack_configurations.return_value + ) + + def test_diff__returned_diff_has_stack_name_of_external_name(self): + diff = self.differ.diff(self.actions) + assert diff.stack_name == self.external_name + + def test_diff__returns_generated_config(self): + diff = self.differ.diff(self.actions) + assert diff.generated_config == self.expected_generated_config + + def test_diff__returns_generated_template(self): + diff = self.differ.diff(self.actions) + assert diff.generated_template == self.actions.generate.return_value + + def test_diff__deployed_stack_exists__returns_is_deployed_as_true(self): + diff = self.differ.diff(self.actions) + assert diff.is_deployed is True + + def test_diff__deployed_stack_does_not_exist__returns_is_deployed_as_false(self): + self.actions.describe.return_value = self.actions.describe.side_effect = None + diff = self.differ.diff(self.actions) + assert diff.is_deployed is False + + def test_diff__deployed_stack_does_not_exist__compares_none_to_generated_config( + self, + ): + self.actions.describe.return_value = self.actions.describe.side_effect = None + self.differ.diff(self.actions) + + self.command_capturer.compare_stack_configurations.assert_called_with( + None, self.expected_generated_config + ) + + def test_diff__deployed_stack_does_not_exist__compares_empty_dict_string_to_generated_template( + self, + ): + self.actions.fetch_remote_template.return_value = None + self.differ.diff(self.actions) + + self.command_capturer.compare_templates.assert_called_with( + "{}", self.actions.generate.return_value + ) + + @pytest.mark.parametrize( + "status", + [ + pytest.param(status) + for status in [ + "CREATE_FAILED", + "ROLLBACK_COMPLETE", + "DELETE_COMPLETE", + ] + ], + ) + def test_diff__non_deployed_stack_status__compares_none_to_generated_config( + self, status + ): + self.stack_status = status + self.differ.diff(self.actions) + + self.command_capturer.compare_stack_configurations.assert_called_with( + None, self.expected_generated_config + ) + + @pytest.mark.parametrize( + "status", + [ + pytest.param(status) + for status in [ + "CREATE_FAILED", + "ROLLBACK_COMPLETE", + "DELETE_COMPLETE", + ] + ], + ) + def test_diff__non_deployed_stack_status__compares_empty_dict_string_to_generated_template( + self, status + ): + self.stack_status = status + self.differ.diff(self.actions) + self.command_capturer.compare_templates.assert_called_with( + "{}", self.actions.generate.return_value + ) + + def test_diff__deployed_stack_has_default_values__doesnt_pass_parameter__compares_identical_configs( + self, + ): + self.deployed_parameters["new"] = "default value" + self.deployed_parameter_defaults["new"] = "default value" + self.differ.diff(self.actions) + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_generated_config, self.expected_generated_config + ) + + def test_diff__deployed_stack_has_list_default_parameter__doesnt_pass_parameter__compares_identical_configs( + self, + ): + self.deployed_parameters["new"] = "first,second,third" + self.deployed_parameter_defaults["new"] = "first, second, third" + self.deployed_parameter_types["new"] = "CommaDelimitedList" + self.differ.diff(self.actions) + + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_generated_config, self.expected_generated_config + ) + + def test_diff__deployed_stack_has_default_values__passes_the_parameter__compares_identical_configs( + self, + ): + self.deployed_parameters["new"] = "default value" + self.deployed_parameter_defaults["new"] = "default value" + self.parameters_on_stack_config["new"] = "default value" + self.differ.diff(self.actions) + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_generated_config, self.expected_generated_config + ) + + def test_diff__deployed_stack_has_default_values__passes_different_value__compares_different_configs( + self, + ): + self.deployed_parameters["new"] = "default value" + self.deployed_parameter_defaults["new"] = "default value" + self.parameters_on_stack_config["new"] = "custom value" + self.differ.diff(self.actions) + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_deployed_config, self.expected_generated_config + ) + + def test_diff__stack_exists_with_same_config_but_template_does_not__compares_identical_configs( + self, + ): + self.actions.fetch_remote_template_summary.side_effect = None + self.actions.fetch_remote_template_summary.return_value = None + self.actions.fetch_remote_template.return_value = None + self.differ.diff(self.actions) + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_generated_config, self.expected_generated_config + ) + + def test_diff__deployed_parameter_has_linebreak_but_otherwise_no_difference__compares_identical_configs( + self, + ): + self.deployed_parameters["param"] = self.deployed_parameters["param"] + "\n" + self.differ.diff(self.actions) + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_generated_config, self.expected_generated_config + ) + + def test_diff__parameter_has_identical_string_linebreak__compares_identical_configs( + self, + ): + self.deployed_parameters["param"] = self.deployed_parameters["param"] + "\n" + self.parameters_on_stack_config["param"] = ( + self.parameters_on_stack_config["param"] + "\n" + ) + + self.differ.diff(self.actions) + generated_config = deepcopy(self.expected_generated_config._asdict()) + generated_parameters = generated_config.pop("parameters") + expected_config = StackConfiguration( + parameters={ + key: value.rstrip("\n") for key, value in generated_parameters.items() + }, + **generated_config, + ) + + self.command_capturer.compare_stack_configurations.assert_called_with( + expected_config, expected_config + ) + + def test_diff__parameter_has_identical_list_linebreaks__compares_identical_configs( + self, + ): + self.deployed_parameter_types["param"] = "CommaDelimitedList" + self.deployed_parameters["param"] = "testing\n,this\n,out\n" + self.parameters_on_stack_config["param"] = ["testing\n", "this\n", "out\n"] + + self.differ.diff(self.actions) + generated_config = deepcopy(self.expected_generated_config._asdict()) + generated_parameters = generated_config.pop("parameters") + generated_parameters["param"] = "testing,this,out" + expected_config = StackConfiguration( + parameters=generated_parameters, + **generated_config, + ) + + self.command_capturer.compare_stack_configurations.assert_called_with( + expected_config, expected_config + ) + + def test_diff__no_echo_default_parameter__generated_stack_doesnt_pass_parameter__compares_identical_configs( + self, + ): + self.deployed_parameters["new"] = "****" + self.deployed_parameter_defaults["new"] = "default value" + self.deployed_no_echo_parameters.append("new") + self.differ.diff(self.actions) + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_generated_config, self.expected_generated_config + ) + + def test_diff__generated_template_has_no_echo_parameter__masks_value(self): + self.parameters_on_stack_config["hide_me"] = "don't look at me!" + self.local_no_echo_parameters.append("hide_me") + + expected_generated_config = self.expected_generated_config + expected_generated_config.parameters[ + "hide_me" + ] = StackDiffer.NO_ECHO_REPLACEMENT + + self.differ.diff(self.actions) + + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_deployed_config, + expected_generated_config, + ) + + def test_diff__generated_template_has_no_echo_parameter__show_no_echo__shows_value( + self, + ): + self.parameters_on_stack_config["hide_me"] = "don't look at me!" + self.local_no_echo_parameters.append("hide_me") + self.differ.show_no_echo = True + self.differ.diff(self.actions) + + self.command_capturer.compare_stack_configurations.assert_called_with( + self.expected_deployed_config, + self.expected_generated_config, + ) + + +class TestDeepDiffStackDiffer: + def setup_method(self, method): + self.differ = DeepDiffStackDiffer() + + self.config1 = StackConfiguration( + stack_name="stack", + parameters={"pk1": "pv1"}, + stack_tags={"tk1": "tv1"}, + notifications=["notification"], + cloudformation_service_role=None, + ) + + self.config2 = StackConfiguration( + stack_name="stack", + parameters={"pk1": "pv1", "pk2": "pv2"}, + stack_tags={"tk1": "tv1"}, + notifications=["notification"], + cloudformation_service_role="new_role", + ) + + self.template_dict_1 = { + "AWSTemplateFormat": "2010-09-09", + "Description": "deployed", + "Parameters": {"pk1": "pv1"}, + "Resources": {}, + } + self.template_dict_2 = { + "AWSTemplateFormat": "2010-09-09", + "Description": "deployed", + "Parameters": {"pk1": "pv1"}, + "Resources": { + "MyBucket": { + "Type": "AWS::S3::Bucket", + "Properties": {"BucketName": "test"}, + } + }, + } + + def test_compare_stack_configurations__returns_deepdiff_of_deployed_and_generated( + self, + ): + comparison = self.differ.compare_stack_configurations( + self.config1, self.config2 + ) + assert comparison.t1 == self.config1 + assert comparison.t2 == self.config2 + + def test_compare_stack_configurations__returned_deepdiff_has_verbosity_of_2(self): + comparison = self.differ.compare_stack_configurations( + self.config1, self.config2 + ) + assert comparison.verbose_level == 2 + + def test_compare_stack_configurations__deployed_is_none__returns_deepdiff_with_none_for_t1( + self, + ): + comparison = self.differ.compare_stack_configurations(None, self.config2) + assert comparison.t1 is None + + @pytest.mark.parametrize( + "t1_serializer, t2_serializer", + [ + pytest.param(json.dumps, json.dumps, id="templates are json"), + pytest.param(yaml.dump, yaml.dump, id="templates are yaml"), + pytest.param(json.dumps, yaml.dump, id="templates are mixed formats"), + ], + ) + def test_compare_templates__templates_are_json__returns_deepdiff_of_dicts( + self, t1_serializer, t2_serializer + ): + template1, template2 = t1_serializer(self.template_dict_1), t2_serializer( + self.template_dict_2 + ) + comparison = self.differ.compare_templates(template1, template2) + assert comparison.t1 == self.template_dict_1 + assert comparison.t2 == self.template_dict_2 + + def test_compare_templates__templates_are_yaml_with_intrinsic_functions__returns_deepdiff_of_dicts( + self, + ): + template = """ + Resources: + MyBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref MyParam + """ + comparison = self.differ.compare_templates(template, template) + + expected = cfn_flip.load_yaml(template) + assert (comparison.t1, comparison.t2) == (expected, expected) + + def test_compare_templates__deployed_is_empty_dict_string__returns_deepdiff_with_empty_dict_for_t1( + self, + ): + template = json.dumps(self.template_dict_1) + comparison = self.differ.compare_templates("{}", template) + assert comparison.t1 == {} + + +class TestDifflibStackDiffer: + def setup_method(self, method): + self.serialize = cfn_flip.dump_yaml + self.differ = DifflibStackDiffer() + self.config1 = StackConfiguration( + stack_name="stack", + parameters={"pk1": "pv1"}, + stack_tags={"tk1": "tv1"}, + notifications=["notification"], + cloudformation_service_role=None, + ) + + self.config2 = StackConfiguration( + stack_name="stack", + parameters={"pk1": "pv1", "pk2": "pv2"}, + stack_tags={"tk1": "tv1"}, + notifications=["notification"], + cloudformation_service_role="new_role", + ) + + self.template_dict_1 = { + "AWSTemplateFormat": "2010-09-09", + "Description": "deployed", + "Parameters": {"pk1": "pv1"}, + "Resources": {}, + } + self.template_dict_2 = { + "AWSTemplateFormat": "2010-09-09", + "Description": "deployed", + "Parameters": {"pk1": "pv1"}, + "Resources": { + "MyBucket": { + "Type": "AWS::S3::Bucket", + "Properties": {"BucketName": "test"}, + } + }, + } + + def create_expected_diff(self, first, second, already_formatted=False): + if not already_formatted: + deployed_dict, deployed_format = cfn_flip.load(first) + generated_dict, generated_format = cfn_flip.load(second) + dumpers = {"json": cfn_flip.dump_json, "yaml": cfn_flip.dump_yaml} + first = dumpers[generated_format](deployed_dict) + second = dumpers[generated_format](generated_dict) + first_list, second_list = first.splitlines(), second.splitlines() + return list( + difflib.unified_diff( + first_list, + second_list, + fromfile="deployed", + tofile="generated", + lineterm="", + ) + ) + + def test_compare_stack_configurations__returns_diff_of_deployed_and_generated_when_converted_to_dicts( + self, + ): + comparison = self.differ.compare_stack_configurations( + self.config1, self.config2 + ) + expected_config_1_dict = self.make_config_comparable(self.config1) + expected_config_2_dict = self.make_config_comparable(self.config2) + expected_config_1 = self.serialize(expected_config_1_dict) + expected_config_2 = self.serialize(expected_config_2_dict) + expected = self.create_expected_diff(expected_config_1, expected_config_2) + + assert comparison == expected + + def make_config_comparable(self, config: StackConfiguration): + config_dict = dict(config._asdict()) + without_empty_values = { + key: value + for key, value in config_dict.items() + if value not in (None, [], {}) and key != "stack_name" + } + return without_empty_values + + def test_compare_stack_configurations__deployed_is_none__returns_diff_with_none( + self, + ): + comparison = self.differ.compare_stack_configurations(None, self.config2) + expected = self.create_expected_diff( + self.serialize(None), + self.serialize(self.make_config_comparable(self.config2)), + ) + assert comparison == expected + + def test_compare_stack_configurations__deployed_is_none__all_configs_are_falsey__returns_diff_with_none( + self, + ): + empty_config = StackConfiguration( + stack_name="stack", + parameters={}, + stack_tags={}, + notifications=[], + cloudformation_service_role=None, + ) + comparison = self.differ.compare_stack_configurations(None, empty_config) + + expected = self.create_expected_diff( + self.serialize(None), + self.serialize(self.make_config_comparable(empty_config)), + already_formatted=True, + ) + assert comparison == expected + + @pytest.mark.parametrize( + "serializer", + [ + pytest.param(json.dumps, id="templates are json"), + pytest.param(yaml.dump, id="templates are yaml"), + ], + ) + def test_compare_templates__templates_are_json__returns_deepdiff_of_dicts( + self, + serializer, + ): + template1, template2 = serializer(self.template_dict_1), serializer( + self.template_dict_2 + ) + comparison = self.differ.compare_templates(template1, template2) + expected = self.create_expected_diff(template1, template2) + assert comparison == expected + + def test_compare_templates__deployed_is_empty_dict_string__returns_diff_with_empty_string( + self, + ): + template = json.dumps(self.template_dict_1) + comparison = self.differ.compare_templates("{}", template) + expected = self.create_expected_diff("{}", template) + assert comparison == expected + + def test_compare_templates__json_template__only_indentation_diff__returns_no_diff( + self, + ): + template1 = json.dumps(self.template_dict_1, indent=2) + template2 = json.dumps(self.template_dict_1, indent=4) + comparison = self.differ.compare_templates(template1, template2) + assert len(comparison) == 0 + + def test_compare_templates__yaml_template__only_indentation_diff__returns_no_diff( + self, + ): + template1 = yaml.dump(self.template_dict_1, indent=2) + template2 = yaml.dump(self.template_dict_1, indent=4) + comparison = self.differ.compare_templates(template1, template2) + assert len(comparison) == 0 + + def test_compare_templates__opposite_template_types_but_identical_template__returns_no_diff( + self, + ): + template1 = json.dumps(self.template_dict_1) + template2 = yaml.dump(self.template_dict_1) + comparison = self.differ.compare_templates(template1, template2) + assert len(comparison) == 0 diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 2f306f796..ba52792aa 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,17 +1,31 @@ # -*- coding: utf-8 -*- +import warnings +import deprecation import pytest from os.path import join, sep +from datetime import datetime, timezone, timedelta from sceptre.exceptions import PathConversionError -from sceptre.helpers import get_external_stack_name +from sceptre.helpers import ( + get_external_stack_name, + create_deprecated_alias_property, + delete_keys_from_containers, +) from sceptre.helpers import normalise_path from sceptre.helpers import sceptreise_path +from sceptre.helpers import extract_datetime_from_aws_response_headers, gen_repr +from sceptre import __version__ -class TestHelpers(object): +class SomeClass(object): + def __init__(self, str_attr: str, int_attr: int): + self.str_attr = str_attr + self.int_attr = int_attr + +class TestHelpers(object): def test_get_external_stack_name(self): result = get_external_stack_name("prj", "dev/ew1/jump-host") assert result == "prj-dev-ew1-jump-host" @@ -21,11 +35,11 @@ def test_normalise_path_with_valid_path(self): assert path == join("valid", "path") def test_normalise_path_with_backslashes_in_path(self): - path = normalise_path("valid\path") + path = normalise_path("valid\\path") assert path == join("valid", "path") def test_normalise_path_with_double_backslashes_in_path(self): - path = normalise_path('valid\\path') + path = normalise_path("valid\\path") assert path == join("valid", "path") def test_normalise_path_with_leading_slash(self): @@ -33,37 +47,150 @@ def test_normalise_path_with_leading_slash(self): assert path == join("{}this".format(sep), "is", "valid") def test_normalise_path_with_leading_backslash(self): - path = normalise_path('\\this\path\is\\valid') + path = normalise_path("\\this\\path\\is\\valid") assert path == join("{}this".format(sep), "path", "is", "valid") def test_normalise_path_with_trailing_slash(self): with pytest.raises(PathConversionError): - normalise_path( - "this/path/is/invalid/" - ) + normalise_path("this/path/is/invalid/") def test_normalise_path_with_trailing_backslash(self): with pytest.raises(PathConversionError): - normalise_path( - 'this\path\is\invalid\\' - ) + normalise_path("this\\path\\is\\invalid\\") def test_sceptreise_path_with_valid_path(self): - path = 'dev/app/stack' + path = "dev/app/stack" assert sceptreise_path(path) == path def test_sceptreise_path_with_windows_path(self): - windows_path = 'dev\\app\\stack' - assert sceptreise_path(windows_path) == 'dev/app/stack' + windows_path = "dev\\app\\stack" + assert sceptreise_path(windows_path) == "dev/app/stack" def test_sceptreise_path_with_trailing_slash(self): with pytest.raises(PathConversionError): - sceptreise_path( - "this/path/is/invalid/" - ) + sceptreise_path("this/path/is/invalid/") def test_sceptreise_path_with_trailing_backslash(self): with pytest.raises(PathConversionError): - sceptreise_path( - 'this\path\is\invalid\\' + sceptreise_path("this\\path\\is\\invalid\\") + + def test_get_response_datetime__response_is_valid__returns_datetime(self): + resp = { + "ResponseMetadata": { + "HTTPHeaders": {"date": "Wed, 16 Oct 2019 07:28:00 GMT"} + } + } + assert extract_datetime_from_aws_response_headers(resp) == datetime( + 2019, 10, 16, 7, 28, tzinfo=timezone.utc + ) + + def test_get_response_datetime__response_has_offset__returns_datetime(self): + resp = { + "ResponseMetadata": { + "HTTPHeaders": {"date": "Wed, 16 Oct 2019 07:28:00 +0400"} + } + } + offset = timezone(timedelta(hours=4)) + assert extract_datetime_from_aws_response_headers(resp) == datetime( + 2019, 10, 16, 7, 28, tzinfo=offset + ) + + def test_get_response_datetime__date_string_is_invalid__returns_none(self): + resp = {"ResponseMetadata": {"HTTPHeaders": {"date": "garbage"}}} + assert extract_datetime_from_aws_response_headers(resp) is None + + def test_get_response_datetime__response_is_empty__returns_none(self): + assert extract_datetime_from_aws_response_headers({}) is None + + def test_get_response_datetime__response_is_none__returns_none(self): + assert extract_datetime_from_aws_response_headers(None) is None + + def test_repr__one_attr__no_commas(self): + i = SomeClass("a", 2) + assert gen_repr(i, attributes=["str_attr"]) == "SomeClass(str_attr='a')" + + def test_repr__two_attrs__correct_order(self): + i = SomeClass("b", 6) + assert ( + gen_repr(i, attributes=["int_attr", "str_attr"]) + == "SomeClass(int_attr=6, str_attr='b')" + ) + + def test_repr__override_label__correct_name(self): + i = SomeClass("q", 123) + assert gen_repr(i, class_label="My.Class") == "My.Class()" + + def test_create_deprecated_alias_property__alias_getter_returns_alias_target_value( + self, + ): + class MyClass: + target = "winner" + alias = create_deprecated_alias_property( + "alias", "target", __version__, None + ) + + obj = MyClass() + + assert obj.alias == obj.target + + def test_create_deprecated_alias_property__alias_setter_returns_alias_target_value( + self, + ): + class MyClass: + target = "loser" + alias = create_deprecated_alias_property( + "alias", "target", __version__, None + ) + + obj = MyClass() + obj.alias = expected = "winner" + assert obj.target == expected + + def test_create_deprecated_alias_property__emits_warning_when_getting_value(self): + class MyClass: + target = "winner" + alias = create_deprecated_alias_property( + "alias", "target", __version__, None ) + + obj = MyClass() + + with warnings.catch_warnings(record=True) as messages: + obj.alias + + assert len(messages) == 1 + assert messages[0].category == deprecation.DeprecatedWarning + + def test_create_deprecated_alias_property__emits_warning_when_setting_value(self): + class MyClass: + target = "loser" + alias = create_deprecated_alias_property( + "alias", "target", __version__, None + ) + + obj = MyClass() + + with warnings.catch_warnings(record=True) as messages: + obj.alias = "winner" + + assert len(messages) == 1 + assert messages[0].category == deprecation.DeprecatedWarning + + def test_delete_keys_from_containers__removes_keys_from_dicts(self): + a = {"keep": "me", "kill": "me"} + b = {"keep": "me", "take": "me out"} + c = {"keep": "me", "destroy": "me"} + + arg = [(a, "kill"), (b, "take"), (c, "destroy")] + delete_keys_from_containers(arg) + expected = {"keep": "me"} + assert a == b == c == expected + + def test_delete_keys_from_containers__removes_indexes_from_lists(self): + a = ["keep me", "kill me", "keep me", "destroy me"] + b = ["take me out", "keep me", "send me the true death", "keep me"] + + arg = [(a, 1), (a, 3), (b, 0), (b, 2)] + delete_keys_from_containers(arg) + expected = ["keep me", "keep me"] + assert a == b == expected diff --git a/tests/test_hooks/test_asg_scaling_processes.py b/tests/test_hooks/test_asg_scaling_processes.py index 99c634728..5a0377178 100644 --- a/tests/test_hooks/test_asg_scaling_processes.py +++ b/tests/test_hooks/test_asg_scaling_processes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import pytest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from sceptre.exceptions import InvalidHookArgumentValueError from sceptre.exceptions import InvalidHookArgumentSyntaxError @@ -14,6 +14,7 @@ class TestASGScalingProcesses(object): def setup_method(self, test_method): self.stack = MagicMock(spec=Stack) + self.stack.name = "my/stack.yaml" self.stack.connection_manager = MagicMock(spec=ConnectionManager) self.stack.external_name = "external_name" self.asg_scaling_processes = ASGScalingProcesses(None, self.stack) @@ -23,18 +24,18 @@ def test_get_stack_resources_sends_correct_request(self): "StackResources": [ { "ResourceType": "AWS::AutoScaling::AutoScalingGroup", - 'PhysicalResourceId': 'cloudreach-examples-asg' + "PhysicalResourceId": "cloudreach-examples-asg", } ] } self.asg_scaling_processes._get_stack_resources() self.stack.connection_manager.call.assert_called_with( - service="cloudformation", - command="describe_stack_resources", - kwargs={ - "StackName": "external_name", - } - ) + service="cloudformation", + command="describe_stack_resources", + kwargs={ + "StackName": "external_name", + }, + ) @patch( "sceptre.hooks.asg_scaling_processes" @@ -43,14 +44,16 @@ def test_get_stack_resources_sends_correct_request(self): def test_find_autoscaling_groups_with_stack_with_asgs( self, mock_get_stack_resources ): - mock_get_stack_resources.return_value = [{ - 'LogicalResourceId': 'AutoScalingGroup', - 'PhysicalResourceId': 'cloudreach-examples-asg', - 'ResourceStatus': 'CREATE_COMPLETE', - 'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', - 'StackId': 'arn:aws:...', - 'StackName': 'cloudreach-examples-dev-vpc' - }] + mock_get_stack_resources.return_value = [ + { + "LogicalResourceId": "AutoScalingGroup", + "PhysicalResourceId": "cloudreach-examples-asg", + "ResourceStatus": "CREATE_COMPLETE", + "ResourceType": "AWS::AutoScaling::AutoScalingGroup", + "StackId": "arn:aws:...", + "StackName": "cloudreach-examples-dev-vpc", + } + ] response = self.asg_scaling_processes._find_autoscaling_groups() assert response == ["cloudreach-examples-asg"] @@ -72,19 +75,17 @@ def test_find_autoscaling_groups_with_stack_without_asgs( ".ASGScalingProcesses._find_autoscaling_groups" ) def test_run_with_resume_argument(self, mock_find_autoscaling_groups): - self.asg_scaling_processes.argument = u"resume::ScheduledActions" + self.asg_scaling_processes.argument = "resume::ScheduledActions" mock_find_autoscaling_groups.return_value = ["autoscaling_group_1"] self.asg_scaling_processes.run() self.stack.connection_manager.call.assert_called_once_with( - service="autoscaling", - command="resume_processes", - kwargs={ - "AutoScalingGroupName": "autoscaling_group_1", - "ScalingProcesses": [ - "ScheduledActions" - ] - } - ) + service="autoscaling", + command="resume_processes", + kwargs={ + "AutoScalingGroupName": "autoscaling_group_1", + "ScalingProcesses": ["ScheduledActions"], + }, + ) @patch( "sceptre.hooks.asg_scaling_processes" @@ -95,24 +96,20 @@ def test_run_with_suspend_argument(self, mock_find_autoscaling_groups): mock_find_autoscaling_groups.return_value = ["autoscaling_group_1"] self.asg_scaling_processes.run() self.stack.connection_manager.call.assert_called_once_with( - service="autoscaling", - command="suspend_processes", - kwargs={ - "AutoScalingGroupName": "autoscaling_group_1", - "ScalingProcesses": [ - "ScheduledActions" - ] - } - ) + service="autoscaling", + command="suspend_processes", + kwargs={ + "AutoScalingGroupName": "autoscaling_group_1", + "ScalingProcesses": ["ScheduledActions"], + }, + ) @patch( "sceptre.hooks.asg_scaling_processes" ".ASGScalingProcesses._find_autoscaling_groups" ) - def test_run_with_invalid_string_argument( - self, mock_find_autoscaling_groups - ): - self.asg_scaling_processes.argument = u"invalid_string" + def test_run_with_invalid_string_argument(self, mock_find_autoscaling_groups): + self.asg_scaling_processes.argument = "invalid_string" mock_find_autoscaling_groups.return_value = ["autoscaling_group_1"] with pytest.raises(InvalidHookArgumentSyntaxError): self.asg_scaling_processes.run() diff --git a/tests/test_hooks/test_cmd.py b/tests/test_hooks/test_cmd.py index bd8bdfbbd..3f15a7f0f 100644 --- a/tests/test_hooks/test_cmd.py +++ b/tests/test_hooks/test_cmd.py @@ -1,31 +1,37 @@ # -*- coding: utf-8 -*- +import subprocess +from unittest.mock import patch, Mock import pytest -from mock import patch -import subprocess -from sceptre.hooks.cmd import Cmd from sceptre.exceptions import InvalidHookArgumentTypeError +from sceptre.hooks.cmd import Cmd +from sceptre.stack import Stack class TestCmd(object): def setup_method(self, test_method): - self.cmd = Cmd() + self.stack = Mock(Stack) + self.stack.name = "my/stack.yaml" + self.cmd = Cmd(stack=self.stack) def test_run_with_non_str_argument(self): self.cmd.argument = None with pytest.raises(InvalidHookArgumentTypeError): self.cmd.run() - @patch('sceptre.hooks.cmd.subprocess.check_call') + @patch("sceptre.hooks.cmd.subprocess.check_call") def test_run_with_str_argument(self, mock_call): - self.cmd.argument = u"echo hello" + self.cmd.argument = "echo hello" self.cmd.run() - mock_call.assert_called_once_with(u"echo hello", shell=True) + expected_envs = ( + self.stack.connection_manager.create_session_environment_variables.return_value + ) + mock_call.assert_called_once_with("echo hello", shell=True, env=expected_envs) - @patch('sceptre.hooks.cmd.subprocess.check_call') + @patch("sceptre.hooks.cmd.subprocess.check_call") def test_run_with_erroring_command(self, mock_call): mock_call.side_effect = subprocess.CalledProcessError(1, "echo") - self.cmd.argument = u"echo hello" + self.cmd.argument = "echo hello" with pytest.raises(subprocess.CalledProcessError): self.cmd.run() diff --git a/tests/test_hooks/test_hooks.py b/tests/test_hooks/test_hooks.py index 3d1497a32..47194b6bc 100644 --- a/tests/test_hooks/test_hooks.py +++ b/tests/test_hooks/test_hooks.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- -from mock import MagicMock +from unittest import TestCase +from unittest.mock import MagicMock, Mock from sceptre.hooks import Hook, HookProperty, add_stack_hooks, execute_hooks +from sceptre.resolvers import Resolver +from sceptre.stack import Stack +import logging class MockHook(Hook): - def __init__(self, *args, **kwargs): - super(MockHook, self).__init__(*args, **kwargs) - def run(self): pass @@ -22,8 +23,8 @@ def test_add_stack_hooks(self): mock_object = MagicMock() mock_object.stack.hooks = { - 'before_mock_function': [mock_hook_before], - 'after_mock_function': [mock_hook_after] + "before_mock_function": [mock_hook_before], + "after_mock_function": [mock_hook_after], } def mock_function(self): @@ -31,7 +32,7 @@ def mock_function(self): assert mock_hook_after.run.call_count == 0 mock_object.mock_function = mock_function - mock_object.mock_function.__name__ = 'mock_function' + mock_object.mock_function.__name__ = "mock_function" add_stack_hooks(mock_object.mock_function)(mock_object) @@ -60,21 +61,36 @@ def test_execute_hooks_with_multiple_hook(self): hook_2.run.called_once_with() -class TestHook(object): - def setup_method(self, test_method): - self.hook = MockHook() +class MyResolver(Resolver): + def resolve(self): + return self.argument + + +class TestHook(TestCase): + def setUp(self): + self.stack = Mock(Stack) + self.stack.name = "my/stack" + self.hook = MockHook(stack=self.stack) - def test_hook_inheritance(self): - assert isinstance(self.hook, Hook) + def test_logger__logs_have_stack_name_prefix(self): + with self.assertLogs(self.hook.logger.name, logging.INFO) as handler: + self.hook.logger.info("Bonjour") + + assert handler.records[0].message == f"{self.stack.name} - Bonjour" + + def test_argument__supports_resolvers_in_arguments(self): + arg = [MyResolver("hello")] + hook = MockHook(arg, self.stack) + self.assertEqual(["hello"], hook.argument) class MockClass(object): hook_property = HookProperty("hook_property") config = MagicMock() + name = "my/stack.yaml" class TestHookPropertyDescriptor(object): - def setup_method(self, test_method): self.mock_object = MockClass() @@ -82,7 +98,9 @@ def test_setting_hook_property(self): mock_hook = MagicMock(spec=MockHook) self.mock_object.hook_property = [mock_hook] - assert self.mock_object._hook_property == [mock_hook] + assert self.mock_object._hook_property == [ + mock_hook.clone_for_stack.return_value + ] def test_getting_hook_property(self): self.mock_object._hook_property = self.mock_object diff --git a/tests/test_plan.py b/tests/test_plan.py index b09e03d1d..9e92ba9ab 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -1,5 +1,5 @@ import pytest -from mock import MagicMock, patch, sentinel +from unittest.mock import MagicMock, patch, sentinel from sceptre.context import SceptreContext from sceptre.stack import Stack @@ -8,28 +8,33 @@ class TestSceptrePlan(object): - def setup_method(self, test_method): self.patcher_SceptrePlan = patch("sceptre.plan.plan.SceptrePlan") self.stack = Stack( - name='dev/app/stack', project_code=sentinel.project_code, - template_path=sentinel.template_path, region=sentinel.region, - profile=sentinel.profile, parameters={"key1": "val1"}, - sceptre_user_data=sentinel.sceptre_user_data, hooks={}, - s3_details=None, dependencies=sentinel.dependencies, - role_arn=sentinel.role_arn, protected=False, - tags={"tag1": "val1"}, external_name=sentinel.external_name, + name="dev/app/stack", + project_code=sentinel.project_code, + template_handler_config={"path": "/path/to/thing"}, + region=sentinel.region, + profile=sentinel.profile, + parameters={"key1": "val1"}, + sceptre_user_data=sentinel.sceptre_user_data, + hooks={}, + s3_details=None, + dependencies=sentinel.dependencies, + cloudformation_service_role=sentinel.cloudformation_service_role, + protected=False, + tags={"tag1": "val1"}, + external_name=sentinel.external_name, notifications=[sentinel.notification], on_failure=sentinel.on_failure, - stack_timeout=sentinel.stack_timeout + stack_timeout=sentinel.stack_timeout, ) self.mock_context = MagicMock(spec=SceptreContext) self.mock_config_reader = MagicMock(spec=ConfigReader) self.mock_context.project_path = sentinel.project_path self.mock_context.command_path = sentinel.command_path self.mock_context.config_file = sentinel.config_file - self.mock_context.full_config_path.return_value =\ - sentinel.full_config_path + self.mock_context.full_config_path.return_value = sentinel.full_config_path self.mock_context.user_variables = {} self.mock_context.options = {} self.mock_context.no_colour = True @@ -47,8 +52,8 @@ def test_planner_executes_with_params(self): plan = MagicMock(spec=SceptrePlan) plan.context = self.mock_context plan.launch.return_value = sentinel.success - result = plan.launch('test-attribute') - plan.launch.assert_called_once_with('test-attribute') + result = plan.launch("test-attribute") + plan.launch.assert_called_once_with("test-attribute") assert result == sentinel.success def test_command_not_found_error_raised(self): diff --git a/tests/test_resolvers/test_cache.py b/tests/test_resolvers/test_cache.py deleted file mode 100644 index 87abe803a..000000000 --- a/tests/test_resolvers/test_cache.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- - - -class TestResolverCache(object): - def setup_method(self, setup_method): - pass - - def test_singleton_cache(self): - pass diff --git a/tests/test_resolvers/test_environment_variable.py b/tests/test_resolvers/test_environment_variable.py index ba078a78a..e6fe9cc5e 100644 --- a/tests/test_resolvers/test_environment_variable.py +++ b/tests/test_resolvers/test_environment_variable.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- -from mock import patch +from unittest.mock import patch from sceptre.resolvers.environment_variable import EnvironmentVariable class TestEnvironmentVariableResolver(object): - def setup_method(self, test_method): - self.environment_variable_resolver = EnvironmentVariable( - argument=None - ) + self.environment_variable_resolver = EnvironmentVariable(argument=None) @patch("sceptre.resolvers.environment_variable.os") def test_resolving_with_set_environment_variable(self, mock_os): diff --git a/tests/test_resolvers/test_file_contents.py b/tests/test_resolvers/test_file_contents.py index 935536a7f..d6be34fa1 100644 --- a/tests/test_resolvers/test_file_contents.py +++ b/tests/test_resolvers/test_file_contents.py @@ -7,14 +7,11 @@ class TestFileContentsResolver(object): - def setup_method(self, test_method): - self.file_contents_resolver = FileContents( - argument=None - ) + self.file_contents_resolver = FileContents(argument=None) def test_resolving_with_existing_file(self): - with tempfile.NamedTemporaryFile(mode='w+') as f: + with tempfile.NamedTemporaryFile(mode="w+") as f: f.write("file contents") f.seek(0) self.file_contents_resolver.argument = f.name diff --git a/tests/test_resolvers/test_join.py b/tests/test_resolvers/test_join.py new file mode 100644 index 000000000..0c8e1e16a --- /dev/null +++ b/tests/test_resolvers/test_join.py @@ -0,0 +1,73 @@ +from unittest.mock import Mock + +import pytest + +from sceptre.exceptions import InvalidResolverArgumentError +from sceptre.resolvers import Resolver +from sceptre.resolvers.join import Join + + +class ArgResolver(Resolver): + def resolve(self): + return self.argument + + +class TestJoin: + def test_resolve__joins_resolver_values_into_single_string(self): + argument = [ + "/", + [ + ArgResolver("first"), + "middle", + ArgResolver("last"), + ], + ] + join = Join(argument, Mock()) + resolved = join.resolve() + expected = "first/middle/last" + assert expected == resolved + + def test_resolve__argument_returns_non_string__casts_it_to_string(self): + argument = [ + "/", + [ + ArgResolver(123), + "other", + ], + ] + join = Join(argument, Mock()) + resolved = join.resolve() + expected = "123/other" + assert expected == resolved + + def test_resolve__argument_returns_none__not_included_in_string(self): + argument = [ + "/", + [ + ArgResolver("first"), + ArgResolver(None), + ArgResolver("last"), + ], + ] + join = Join(argument, Mock()) + resolved = join.resolve() + expected = "first/last" + assert expected == resolved + + @pytest.mark.parametrize( + "bad_argument", + [ + pytest.param("just a string", id="just a string"), + pytest.param([123, ["something"]], id="non-string delimiter"), + pytest.param( + ["join", "not list to join"], id="second argument is not list" + ), + pytest.param(["first", ["second"], "third"], id="too many items"), + ], + ) + def test_resolve__invalid_arguments_passed__raises_invalid_resolver_argument_error( + self, bad_argument + ): + join = Join(bad_argument, Mock()) + with pytest.raises(InvalidResolverArgumentError): + join.resolve() diff --git a/tests/test_resolvers/test_placeholders.py b/tests/test_resolvers/test_placeholders.py new file mode 100644 index 000000000..236a19e29 --- /dev/null +++ b/tests/test_resolvers/test_placeholders.py @@ -0,0 +1,97 @@ +import pytest + +from sceptre.resolvers import are_placeholders_enabled, Resolver +from sceptre.resolvers.placeholders import ( + use_resolver_placeholders_on_error, + PlaceholderType, + create_placeholder_value, +) + + +class MyResolver(Resolver): + def resolve(self): + return self.argument + + +class TestPlaceholders: + def test_are_placeholders_enabled__returns_false(self): + assert are_placeholders_enabled() is False + + def test_are_placeholders_enabled__in_placeholder_context__returns_true(self): + with use_resolver_placeholders_on_error(): + assert are_placeholders_enabled() is True + + def test_are_placeholders_enabled__out_of_placeholder_context__returns_false(self): + with use_resolver_placeholders_on_error(): + pass + + assert are_placeholders_enabled() is False + + def test_are_placeholders_enabled__error_in_placeholder_context__returns_false( + self, + ): + with pytest.raises(ValueError), use_resolver_placeholders_on_error(): + raise ValueError() + + assert are_placeholders_enabled() is False + + @pytest.mark.parametrize( + "placeholder_type,argument,expected", + [ + pytest.param( + PlaceholderType.explicit, + None, + "{ !MyResolver }", + id="explicit no argument", + ), + pytest.param( + PlaceholderType.explicit, + "argument", + "{ !MyResolver(argument) }", + id="explicit string argument", + ), + pytest.param( + PlaceholderType.explicit, + {"key": "value"}, + "{ !MyResolver({'key': 'value'}) }", + id="explicit dict argument", + ), + pytest.param( + PlaceholderType.explicit, + {"some_key": MyResolver("some value")}, + "{ !MyResolver({'some_key': !MyResolver(some value)}) }", + id="explicit with nested resolvers", + ), + pytest.param( + PlaceholderType.alphanum, None, "MyResolver", id="alphanum no argument" + ), + pytest.param( + PlaceholderType.alphanum, + "argument", + "MyResolverargument", + id="alphanum string argument", + ), + pytest.param( + PlaceholderType.alphanum, + {"key": "value"}, + "MyResolverkeyvalue", + id="alphanum dict argument", + ), + pytest.param( + PlaceholderType.alphanum, + {"some_key": MyResolver("some value")}, + "MyResolversomekeyMyResolversomevalue", + id="alphanum with nested resolvers", + ), + pytest.param( + PlaceholderType.none, "something", None, id="none type placeholder type" + ), + ], + ) + def test_create_placeholder_value(self, placeholder_type, argument, expected): + resolver = MyResolver(argument) + + with use_resolver_placeholders_on_error(): + result = create_placeholder_value(resolver, placeholder_type) + + assert result == expected diff --git a/tests/test_resolvers/test_resolver.py b/tests/test_resolvers/test_resolver.py index 7e0e1bf8b..111661eda 100644 --- a/tests/test_resolvers/test_resolver.py +++ b/tests/test_resolvers/test_resolver.py @@ -1,8 +1,22 @@ # -*- coding: utf-8 -*- +import logging +from unittest import TestCase +from unittest.mock import call, Mock, sentinel, MagicMock -from mock import sentinel, MagicMock +import pytest -from sceptre.resolvers import Resolver, ResolvableProperty +from sceptre.exceptions import InvalidResolverArgumentError +from sceptre.resolvers import ( + Resolver, + ResolvableContainerProperty, + ResolvableValueProperty, + RecursiveResolve, +) +from sceptre.resolvers.placeholders import ( + use_resolver_placeholders_on_error, + create_placeholder_value, + PlaceholderType, +) class MockResolver(Resolver): @@ -12,36 +26,207 @@ class MockResolver(Resolver): Resolver, which is not otherwise instantiable. """ + setup_has_been_called = False + def resolve(self): pass + def setup(self): + self.setup_has_been_called = True + + +class NestedResolver(Resolver): + setup_has_been_called = False + resolve_count = 0 + + def setup(self): + self.setup_has_been_called = True + + def resolve(self): + self.resolve_count += 1 + return self.argument + class MockClass(object): - resolvable_property = ResolvableProperty("resolvable_property") + resolvable_container_property = ResolvableContainerProperty( + "resolvable_container_property" + ) + container_with_alphanum_placeholder = ResolvableContainerProperty( + "container_with_placeholder_override", PlaceholderType.alphanum + ) + resolvable_value_property = ResolvableValueProperty("resolvable_value_property") + value_with_none_placeholder = ResolvableValueProperty( + "value_with_placeholder_override", PlaceholderType.none + ) config = MagicMock() + name = "my/stack" -class TestResolver(object): - def setup_method(self, test_method): +class TestResolver(TestCase): + def setUp(self): self.mock_resolver = MockResolver( - argument=sentinel.argument, - stack=sentinel.stack + argument=sentinel.argument, stack=sentinel.stack ) def test_init(self): assert self.mock_resolver.stack == sentinel.stack assert self.mock_resolver.argument == sentinel.argument + def test_logger__logs_have_stack_name_prefix(self): + with self.assertLogs(self.mock_resolver.logger.name, logging.INFO) as handler: + self.mock_resolver.logger.info("Bonjour") + + assert handler.records[0].message == f"{sentinel.stack.name} - Bonjour" + + def test_invalid_argument_error__raises_invalid_argument_error_with_stack_name_prefix( + self, + ): + with pytest.raises( + InvalidResolverArgumentError, match=f"{sentinel.stack.name} - danger" + ): + self.mock_resolver.raise_invalid_argument_error("danger") + + +class TestCustomYamlTagBase(TestCase): + def setUp(self): + self.stack = Mock() + self.stack.name = "My Stack" + + def test_clone_for_stack__dict_argument_is_cloned(self): + arg = {"greeting": "hello"} + resolver = MockResolver(arg) + clone = resolver.clone_for_stack(self.stack) + self.assertIsNot(clone.argument, arg) + + def test_clone_for_stack__list_argument_is_cloned(self): + arg = ["hello"] + resolver = MockResolver(arg) + clone = resolver.clone_for_stack(self.stack) + self.assertIsNot(clone.argument, arg) + + def test_clone_for_stack__nested_dict_argument_is_cloned(self): + arg = {"greetings": {"French": "bonjour"}} + resolver = MockResolver(arg) + clone = resolver.clone_for_stack(None) + self.assertIsNot(clone.argument["greetings"], arg["greetings"]) + + def test_clone_for_stack__nested_list_argument_is_cloned(self): + arg = [{"French": "bonjour"}] + resolver = MockResolver(arg) + clone = resolver.clone_for_stack(None) + self.assertIsNot(clone.argument[0], arg[0]) + + def test_clone_for_stack__nested_resolvers_are_cloned(self): + arg = {"greetings": {"French": NestedResolver("bonjour")}} + resolver = MockResolver(arg) + clone = resolver.clone_for_stack(None) + self.assertIsNot( + clone.argument["greetings"]["French"], arg["greetings"]["French"] + ) + + def test_clone_for_stack__calls_setup(self): + arg = {"greetings": {"French": NestedResolver("bonjour")}} + resolver = MockResolver(arg) + clone = resolver.clone_for_stack(self.stack) + self.assertTrue(clone.setup_has_been_called) + + def test_clone_for_stack__nested_resolvers_are_setup(self): + arg = {"greetings": {"French": NestedResolver("bonjour")}} + resolver = MockResolver(arg) + # Passing None here will mean that the argument won't actually be resolved, so it lets us + # access the actual resolver instance. + clone = resolver.clone_for_stack(None) -class TestResolvablePropertyDescriptor(object): + self.assertTrue(clone.argument["greetings"]["French"].setup_has_been_called) + def test_clone_for_stack__resolvers_inside_resolvers_are_cloned(self): + arg = { + "greetings": { + "French": NestedResolver({"informal": NestedResolver("salut")}) + } + } + resolver = MockResolver(arg) + # Passing None here will mean that the argument won't actually be resolved, so it lets us + # access the actual resolver instance. + clone = resolver.clone_for_stack(None) + self.assertIsNot( + clone.argument["greetings"]["French"].argument["informal"], + arg["greetings"]["French"].argument["informal"], + ) + + def test_clone_for_stack__resolvers_inside_resolvers_are_setup(self): + arg = { + "greetings": { + "French": NestedResolver({"informal": NestedResolver("salut")}) + } + } + resolver = MockResolver(arg) + # Passing None here will mean that the argument won't actually be resolved, so it lets us + # access the actual resolver instance. + clone = resolver.clone_for_stack(None) + self.assertTrue( + clone.argument["greetings"]["French"] + .argument["informal"] + .setup_has_been_called + ) + + def test_argument__has_stack__arg_is_string__resolves_to_string(self): + arg = "hello" + resolver = MockResolver(arg) + clone = resolver.clone_for_stack(self.stack) + + self.assertEqual("hello", clone.argument) + + def test_argument__cloned_for_stack__resolves_arg_resolver(self): + arg = {"greetings": {"French": NestedResolver("bonjour")}} + resolver = MockResolver(arg).clone_for_stack(self.stack) + expected = {"greetings": {"French": "bonjour"}} + self.assertEqual(expected, resolver.argument) + + def test_argument__cloned_for_stack__nested_argument_in_nested_argument__results_all_resolvers( + self, + ): + arg = { + "greetings": { + "French": NestedResolver({"informal": NestedResolver("salut")}) + } + } + resolver = MockResolver(arg).clone_for_stack(self.stack) + expected = {"greetings": {"French": {"informal": "salut"}}} + self.assertEqual(expected, resolver.argument) + + def test_argument__cloned_for_stack__nested_argument_in_dict_resolves_to_nothing__removes_it_from_argument( + self, + ): + arg = {"greetings": {"French": NestedResolver(None)}} + resolver = MockResolver(arg).clone_for_stack(self.stack) + expected = {"greetings": {}} + self.assertEqual(expected, resolver.argument) + + def test_argument__nested_argument_in_list_resolves_to_nothing__removes_it_from_argument( + self, + ): + arg = { + "greetings": [ + NestedResolver(None), + "Hello", + NestedResolver(None), + "Bonjour", + ] + } + resolver = MockResolver(arg).clone_for_stack(self.stack) + expected = {"greetings": ["Hello", "Bonjour"]} + self.assertEqual(expected, resolver.argument) + + +class TestResolvableContainerPropertyDescriptor: def setup_method(self, test_method): self.mock_object = MockClass() def test_setting_resolvable_property_with_none(self): - self.mock_object.resolvable_property = None - assert self.mock_object._resolvable_property is None + self.mock_object.resolvable_container_property = None + assert self.mock_object._resolvable_container_property is None def test_setting_resolvable_property_with_nested_lists(self): mock_resolver = MagicMock(spec=MockResolver) @@ -52,25 +237,32 @@ def test_setting_resolvable_property_with_nested_lists(self): [ mock_resolver, "String", + [[mock_resolver, "String", None], mock_resolver, "String"], + ], + ] + + cloned_data_structure = [ + "String", + mock_resolver.clone_for_stack.return_value, + [ + mock_resolver.clone_for_stack.return_value, + "String", [ - [ - mock_resolver, - "String", - None - ], - mock_resolver, - "String" - ] - ] + [mock_resolver.clone_for_stack.return_value, "String", None], + mock_resolver.clone_for_stack.return_value, + "String", + ], + ], ] - self.mock_object.resolvable_property = complex_data_structure - assert self.mock_object._resolvable_property == complex_data_structure - assert mock_resolver.stack == self.mock_object + self.mock_object.resolvable_container_property = complex_data_structure + assert self.mock_object._resolvable_container_property == cloned_data_structure + expected_calls = [call(self.mock_object)] * 4 + mock_resolver.clone_for_stack.assert_has_calls(expected_calls) def test_getting_resolvable_property_with_none(self): - self.mock_object._resolvable_property = None - assert self.mock_object.resolvable_property is None + self.mock_object._resolvable_container_property = None + assert self.mock_object.resolvable_container_property is None def test_getting_resolvable_property_with_nested_lists(self): mock_resolver = MagicMock(spec=MockResolver) @@ -82,18 +274,10 @@ def test_getting_resolvable_property_with_nested_lists(self): [ mock_resolver, "String", - [ - [ - mock_resolver, - "String", - None - ], - mock_resolver, - "String" - ], - None + [[mock_resolver, "String", None], mock_resolver, "String"], + None, ], - None + None, ] resolved_complex_data_structure = [ @@ -102,27 +286,17 @@ def test_getting_resolvable_property_with_nested_lists(self): [ "Resolved", "String", - [ - [ - "Resolved", - "String", - None - ], - "Resolved", - "String" - ], - None + [["Resolved", "String", None], "Resolved", "String"], + None, ], - None + None, ] - self.mock_object._resolvable_property = complex_data_structure - prop = self.mock_object.resolvable_property + self.mock_object._resolvable_container_property = complex_data_structure + prop = self.mock_object.resolvable_container_property assert prop == resolved_complex_data_structure - def test_getting_resolvable_property_with_nested_dictionaries_and_lists( - self - ): + def test_getting_resolvable_property_with_nested_dictionaries_and_lists(self): mock_resolver = MagicMock(spec=MockResolver) mock_resolver.resolve.return_value = "Resolved" @@ -131,42 +305,28 @@ def test_getting_resolvable_property_with_nested_dictionaries_and_lists( "None": None, "Resolver": mock_resolver, "List": [ - [ - mock_resolver, - "String", - None - ], - { - "Dictionary": {}, - "String": "String", - "None": None, - "Resolver": mock_resolver, - "List": [ - mock_resolver - ] - }, - mock_resolver, - "String" + [mock_resolver, "String", None], + { + "Dictionary": {}, + "String": "String", + "None": None, + "Resolver": mock_resolver, + "List": [mock_resolver], + }, + mock_resolver, + "String", ], "Dictionary": { "Resolver": mock_resolver, "Dictionary": { - "List": [ - [ - mock_resolver, - "String", - None - ], - mock_resolver, - "String" - ], + "List": [[mock_resolver, "String", None], mock_resolver, "String"], "String": "String", "None": None, - "Resolver": mock_resolver + "Resolver": mock_resolver, }, "String": "String", - "None": None - } + "None": None, + }, } resolved_complex_data_structure = { @@ -174,46 +334,32 @@ def test_getting_resolvable_property_with_nested_dictionaries_and_lists( "None": None, "Resolver": "Resolved", "List": [ - [ - "Resolved", - "String", - None - ], - { - "Dictionary": {}, - "String": "String", - "None": None, - "Resolver": "Resolved", - "List": [ - "Resolved" - ] - }, - "Resolved", - "String" + ["Resolved", "String", None], + { + "Dictionary": {}, + "String": "String", + "None": None, + "Resolver": "Resolved", + "List": ["Resolved"], + }, + "Resolved", + "String", ], "Dictionary": { "Resolver": "Resolved", "Dictionary": { - "List": [ - [ - "Resolved", - "String", - None - ], - "Resolved", - "String" - ], + "List": [["Resolved", "String", None], "Resolved", "String"], "String": "String", "None": None, - "Resolver": "Resolved" + "Resolver": "Resolved", }, "String": "String", - "None": None - } + "None": None, + }, } - self.mock_object._resolvable_property = complex_data_structure - prop = self.mock_object.resolvable_property + self.mock_object._resolvable_container_property = complex_data_structure + prop = self.mock_object.resolvable_container_property assert prop == resolved_complex_data_structure def test_getting_resolvable_property_with_nested_dictionaries(self): @@ -230,11 +376,11 @@ def test_getting_resolvable_property_with_nested_dictionaries(self): "Dictionary": {}, "String": "String", "None": None, - "Resolver": mock_resolver + "Resolver": mock_resolver, }, "String": "String", - "None": None - } + "None": None, + }, } resolved_complex_data_structure = { @@ -247,13 +393,335 @@ def test_getting_resolvable_property_with_nested_dictionaries(self): "Dictionary": {}, "String": "String", "None": None, - "Resolver": "Resolved" + "Resolver": "Resolved", }, "String": "String", - "None": None - } + "None": None, + }, } - self.mock_object._resolvable_property = complex_data_structure - prop = self.mock_object.resolvable_property + self.mock_object._resolvable_container_property = complex_data_structure + prop = self.mock_object.resolvable_container_property assert prop == resolved_complex_data_structure + + def test_get__resolver_references_same_property_for_other_value__resolves_it(self): + class MyResolver(Resolver): + def resolve(self): + return self.stack.resolvable_container_property["other_value"] + + resolver = MyResolver() + self.mock_object.resolvable_container_property = { + "other_value": "abc", + "resolver": resolver, + } + + assert self.mock_object.resolvable_container_property["resolver"] == "abc" + + def test_get__resolver_references_itself__raises_recursive_resolve(self): + class RecursiveResolver(Resolver): + def resolve(self): + return self.stack.resolvable_container_property["resolver"] + + resolver = RecursiveResolver() + self.mock_object.resolvable_container_property = {"resolver": resolver} + with pytest.raises(RecursiveResolve): + self.mock_object.resolvable_container_property + + def test_get__resolvable_container_property_references_same_property_of_other_stack__resolves( + self, + ): + stack1 = MockClass() + stack1.resolvable_container_property = {"testing": "stack1"} + + class OtherStackResolver(Resolver): + def resolve(self): + return stack1.resolvable_container_property["testing"] + + stack2 = MockClass() + stack2.resolvable_container_property = {"resolver": OtherStackResolver()} + + assert stack2.resolvable_container_property == {"resolver": "stack1"} + + def test_get__resolver_resolves_to_none__value_is_dict__deletes_those_items_from_dict( + self, + ): + class MyResolver(Resolver): + def resolve(self): + return None + + resolver = MyResolver() + self.mock_object.resolvable_container_property = { + "a": 4, + "b": resolver, + "c": 3, + "d": resolver, + "e": resolver, + "f": 5, + } + expected = {"a": 4, "c": 3, "f": 5} + assert self.mock_object.resolvable_container_property == expected + + def test_get__resolver_resolves_to_none__value_is_dict__deletes_those_items_from_complex_structure( + self, + ): + class MyResolver(Resolver): + def resolve(self): + return None + + resolver = MyResolver() + self.mock_object.resolvable_container_property = { + "a": 4, + "b": [ + resolver, + ], + "c": [{"v": resolver}], + "d": 3, + } + expected = {"a": 4, "b": [], "c": [{}], "d": 3} + assert self.mock_object.resolvable_container_property == expected + + def test_get__resolver_resolves_to_none__value_is_list__deletes_that_item_from_list( + self, + ): + class MyResolver(Resolver): + def resolve(self): + return None + + resolver = MyResolver() + self.mock_object.resolvable_container_property = [1, resolver, 3] + expected = [1, 3] + assert self.mock_object.resolvable_container_property == expected + + def test_get__resolver_resolves_to_none__value_is_dict__deletes_that_key_from_dict( + self, + ): + class MyResolver(Resolver): + def resolve(self): + return None + + resolver = MyResolver() + self.mock_object.resolvable_container_property = { + "some key": "some value", + "resolver": resolver, + } + expected = {"some key": "some value"} + assert self.mock_object.resolvable_container_property == expected + + def test_get__resolvers_resolves_to_none__value_is_list__deletes_those_items_from_list( + self, + ): + class MyResolver(Resolver): + def resolve(self): + return None + + resolver = MyResolver() + + self.mock_object.resolvable_container_property = [ + 1, + resolver, + 3, + resolver, + resolver, + 6, + ] + expected = [1, 3, 6] + assert self.mock_object.resolvable_container_property == expected + + def test_get__resolvers_resolves_to_none__value_is_list__deletes_all_items_from_list( + self, + ): + class MyResolver(Resolver): + def resolve(self): + return None + + resolver = MyResolver() + + self.mock_object.resolvable_container_property = [resolver, resolver, resolver] + expected = [] + assert self.mock_object.resolvable_container_property == expected + + def test_get__value_in_list_is_none__returns_list_with_none(self): + self.mock_object.resolvable_container_property = [1, None, 3] + expected = [1, None, 3] + assert self.mock_object.resolvable_container_property == expected + + def test_get__value_in_dict_is_none__returns_dict_with_none(self): + self.mock_object.resolvable_container_property = { + "some key": "some value", + "none key": None, + } + expected = {"some key": "some value", "none key": None} + assert self.mock_object.resolvable_container_property == expected + + def test_get__resolver_raises_error__placeholders_allowed__returns_placeholder( + self, + ): + class ErroringResolver(Resolver): + def resolve(self): + raise ValueError() + + resolver = ErroringResolver() + self.mock_object.resolvable_container_property = {"resolver": resolver} + with use_resolver_placeholders_on_error(): + result = self.mock_object.resolvable_container_property + + assert result == { + "resolver": create_placeholder_value(resolver, PlaceholderType.explicit) + } + + def test_get__resolver_raises_error__placeholders_not_allowed__raises_error(self): + class ErroringResolver(Resolver): + def resolve(self): + raise ValueError() + + resolver = ErroringResolver() + self.mock_object.resolvable_container_property = {"resolver": resolver} + with pytest.raises(ValueError): + self.mock_object.resolvable_container_property + + def test_get__resolver_raises_recursive_resolve__placeholders_allowed__raises_error( + self, + ): + class RecursiveResolver(Resolver): + def resolve(self): + raise RecursiveResolve() + + resolver = RecursiveResolver() + self.mock_object.resolvable_container_property = {"resolver": resolver} + with use_resolver_placeholders_on_error(), pytest.raises(RecursiveResolve): + self.mock_object.resolvable_container_property + + def test_get__resolver_raises_error__placeholders_allowed__alternate_placeholder_type__uses_alternate( + self, + ): + class ErroringResolver(Resolver): + def resolve(self): + raise ValueError() + + resolver = ErroringResolver() + self.mock_object.container_with_alphanum_placeholder = {"resolver": resolver} + with use_resolver_placeholders_on_error(): + result = self.mock_object.container_with_alphanum_placeholder + + assert result == { + "resolver": create_placeholder_value(resolver, PlaceholderType.alphanum) + } + + +class TestResolvableValueProperty: + def setup_method(self, test_method): + self.mock_object = MockClass() + + @pytest.mark.parametrize("value", ["string", True, 123, 1.23, None]) + def test_set__non_resolver__sets_private_variable_as_value(self, value): + self.mock_object.resolvable_value_property = value + assert self.mock_object._resolvable_value_property == value + + def test_set__resolver__sets_private_variable_with_clone_of_resolver_with_instance( + self, + ): + resolver = Mock(spec=MockResolver) + self.mock_object.resolvable_value_property = resolver + assert ( + self.mock_object._resolvable_value_property + == resolver.clone_for_stack.return_value + ) + + @pytest.mark.parametrize("value", ["string", True, 123, 1.23, None]) + def test_get__non_resolver__returns_value(self, value): + self.mock_object._resolvable_value_property = value + assert self.mock_object.resolvable_value_property == value + + def test_get__resolver__returns_resolved_value(self): + resolver = Mock(spec=MockResolver) + self.mock_object._resolvable_value_property = resolver + assert ( + self.mock_object.resolvable_value_property == resolver.resolve.return_value + ) + + def test_get__resolver__updates_set_value_with_resolved_value(self): + resolver = Mock(spec=MockResolver) + self.mock_object._resolvable_value_property = resolver + self.mock_object.resolvable_value_property + assert ( + self.mock_object._resolvable_value_property == resolver.resolve.return_value + ) + + def test_get__resolver__resolver_attempts_to_access_resolver__raises_recursive_resolve( + self, + ): + class RecursiveResolver(Resolver): + def resolve(self): + # This should blow up! + self.stack.resolvable_value_property + + resolver = RecursiveResolver() + self.mock_object.resolvable_value_property = resolver + + with pytest.raises(RecursiveResolve): + self.mock_object.resolvable_value_property + + def test_get__resolvable_value_property_references_same_property_of_other_stack__resolves( + self, + ): + stack1 = MockClass() + stack1.resolvable_value_property = "stack1" + + class OtherStackResolver(Resolver): + def resolve(self): + return stack1.resolvable_value_property + + stack2 = MockClass() + stack2.resolvable_value_property = OtherStackResolver() + + assert stack2.resolvable_value_property == "stack1" + + def test_get__resolver_raises_error__placeholders_allowed__returns_placeholder( + self, + ): + class ErroringResolver(Resolver): + def resolve(self): + raise ValueError() + + resolver = ErroringResolver() + self.mock_object.resolvable_value_property = resolver + with use_resolver_placeholders_on_error(): + result = self.mock_object.resolvable_value_property + + assert result == create_placeholder_value(resolver, PlaceholderType.explicit) + + def test_get__resolver_raises_error__placeholders_not_allowed__raises_error(self): + class ErroringResolver(Resolver): + def resolve(self): + raise ValueError() + + resolver = ErroringResolver() + self.mock_object.resolvable_value_property = resolver + with pytest.raises(ValueError): + self.mock_object.resolvable_value_property + + def test_get__resolver_raises_recursive_resolve__placeholders_allowed__raises_error( + self, + ): + class RecursiveResolver(Resolver): + def resolve(self): + raise RecursiveResolve() + + resolver = RecursiveResolver() + self.mock_object.resolvable_value_property = resolver + with use_resolver_placeholders_on_error(), pytest.raises(RecursiveResolve): + self.mock_object.resolvable_value_property + + def test_get__resolver_raises_error__placeholders_allowed__alternate_placeholder_type__uses_alternate_type( + self, + ): + class ErroringResolver(Resolver): + def resolve(self): + raise ValueError() + + resolver = ErroringResolver() + self.mock_object.value_with_none_placeholder = resolver + with use_resolver_placeholders_on_error(): + result = self.mock_object.value_with_none_placeholder + + assert result == create_placeholder_value(resolver, PlaceholderType.none) diff --git a/tests/test_resolvers/test_select.py b/tests/test_resolvers/test_select.py new file mode 100644 index 000000000..c30fb83ee --- /dev/null +++ b/tests/test_resolvers/test_select.py @@ -0,0 +1,68 @@ +from unittest.mock import Mock + +import pytest + +from sceptre.exceptions import InvalidResolverArgumentError +from sceptre.resolvers import Resolver +from sceptre.resolvers.select import Select + + +class MyListResolver(Resolver): + def resolve(self): + return ["first", "second", "third"] + + +class ItemResolver(Resolver): + def resolve(self): + return self.argument + + +class TestSelect: + def test_resolve__second_arg_is_list_resolver__selects_item_at_list_index(self): + argument = [1, MyListResolver()] + select = Select(argument, Mock()) + resolved = select.resolve() + expected = "second" + assert expected == resolved + + def test_resolve__second_arg_is_list_of_resolvers__selects_item_at_list_index(self): + argument = [ + 1, + [ItemResolver("first"), ItemResolver("second"), ItemResolver("third")], + ] + select = Select(argument, Mock()) + resolved = select.resolve() + expected = "second" + assert expected == resolved + + def test_resolve__negative_index__selects_in_reverse(self): + argument = [-1, MyListResolver()] + select = Select(argument, Mock()) + resolved = select.resolve() + expected = "third" + assert expected == resolved + + def test_resolve__can_select_key_from_dict(self): + argument = ["something", ItemResolver({"something": 123})] + select = Select(argument, Mock()) + resolved = select.resolve() + expected = 123 + assert expected == resolved + + @pytest.mark.parametrize( + "bad_argument", + [ + pytest.param("just a string", id="just a string"), + pytest.param([123, "something"], id="second item is not list or dict"), + pytest.param([99, [1, 2]], id="index out of bounds"), + pytest.param(["hello", [1, 2]], id="string index on list"), + pytest.param(["hello", {"something": "else"}], id="key not present"), + pytest.param(["first", ["second"], "third"], id="too many items"), + ], + ) + def test_resolve__invalid_arguments__raises_invalid_resolver_argument_error( + self, bad_argument + ): + select = Select(bad_argument, Mock()) + with pytest.raises(InvalidResolverArgumentError): + select.resolve() diff --git a/tests/test_resolvers/test_split.py b/tests/test_resolvers/test_split.py new file mode 100644 index 000000000..d00ef473c --- /dev/null +++ b/tests/test_resolvers/test_split.py @@ -0,0 +1,36 @@ +from unittest.mock import Mock + +import pytest + +from sceptre.exceptions import InvalidResolverArgumentError +from sceptre.resolvers import Resolver +from sceptre.resolvers.split import Split + + +class MyResolver(Resolver): + def resolve(self): + return "first,second,third" + + +class TestSplit: + def test_resolve__splits_resolver_value_into_list(self): + argument = [",", MyResolver()] + split = Split(argument, Mock()) + resolved = split.resolve() + expected = ["first", "second", "third"] + assert expected == resolved + + @pytest.mark.parametrize( + "bad_argument", + [ + pytest.param("just a string", id="just a string"), + pytest.param([123, "something"], id="first item is not string"), + pytest.param(["something", 123], id="second item is not string"), + ], + ) + def test_resolve__invalid_arguments__raises_invalid_resolver_argument_error( + self, bad_argument + ): + split = Split(bad_argument, Mock()) + with pytest.raises(InvalidResolverArgumentError): + split.resolve() diff --git a/tests/test_resolvers/test_stack_attr.py b/tests/test_resolvers/test_stack_attr.py new file mode 100644 index 000000000..a67510b87 --- /dev/null +++ b/tests/test_resolvers/test_stack_attr.py @@ -0,0 +1,65 @@ +from unittest.mock import Mock + +import pytest + +from sceptre.resolvers.stack_attr import StackAttr +from sceptre.stack import Stack + + +class TestResolver(object): + def setup_method(self, test_method): + self.stack_group_config = {} + self.stack = Mock(spec=Stack, stack_group_config=self.stack_group_config) + self.stack.name = "my/stack.yaml" + + self.resolver = StackAttr(stack=self.stack) + + def test__resolve__returns_attribute_off_stack(self): + self.resolver.argument = "testing_this" + self.stack.testing_this = "hurray!" + result = self.resolver.resolve() + + assert result == "hurray!" + + def test_resolve__nested_attribute__accesses_nested_value(self): + self.stack.testing_this = {"top": [{"thing": "first"}, {"thing": "second"}]} + + self.resolver.argument = "testing_this.top.1.thing" + result = self.resolver.resolve() + + assert result == "second" + + def test_resolve__attribute_not_defined__accesses_it_off_stack_group_config(self): + self.stack.stack_group_config["testing_this"] = { + "top": [{"thing": "first"}, {"thing": "second"}] + } + + self.resolver.argument = "testing_this.top.1.thing" + result = self.resolver.resolve() + + assert result == "second" + + @pytest.mark.parametrize( + "config,attr_name", + [ + ("template", "template_handler_config"), + ("protect", "protected"), + ("stack_name", "external_name"), + ("stack_tags", "tags"), + ], + ) + def test_resolve__accessing_attribute_renamed_on_stack__resolves_correct_value( + self, config, attr_name + ): + setattr(self.stack, attr_name, "value") + self.resolver.argument = config + + result = self.resolver.resolve() + + assert result == "value" + + def test_resolve__attribute_not_defined__raises_attribute_error(self): + self.resolver.argument = "nonexistant" + + with pytest.raises(AttributeError): + self.resolver.resolve() diff --git a/tests/test_resolvers/test_stack_output.py b/tests/test_resolvers/test_stack_output.py index 6ba0fefff..60f437ce3 100644 --- a/tests/test_resolvers/test_stack_output.py +++ b/tests/test_resolvers/test_stack_output.py @@ -1,25 +1,26 @@ # -*- coding: utf-8 -*- import pytest -from mock import MagicMock, patch, sentinel +from unittest.mock import MagicMock, patch, sentinel from sceptre.exceptions import DependencyStackMissingOutputError from sceptre.exceptions import StackDoesNotExistError from botocore.exceptions import ClientError from sceptre.connection_manager import ConnectionManager -from sceptre.resolvers.stack_output import \ - StackOutput, StackOutputExternal, StackOutputBase +from sceptre.resolvers.stack_output import ( + StackOutput, + StackOutputExternal, + StackOutputBase, +) from sceptre.stack import Stack class TestStackOutputResolver(object): - - @patch( - "sceptre.resolvers.stack_output.StackOutput._get_output_value" - ) + @patch("sceptre.resolvers.stack_output.StackOutput._get_output_value") def test_resolver(self, mock_get_output_value): stack = MagicMock(spec=Stack) + stack.name = "my/stack" stack.dependencies = [] stack.project_code = "project-code" stack._connection_manager = MagicMock(spec=ConnectionManager) @@ -29,13 +30,11 @@ def test_resolver(self, mock_get_output_value): dependency.name = "account/dev/vpc" dependency.profile = "dependency_profile" dependency.region = "dependency_region" - dependency.iam_role = "dependency_iam_role" + dependency.sceptre_role = "dependency_sceptre_role" mock_get_output_value.return_value = "output_value" - stack_output_resolver = StackOutput( - "account/dev/vpc.yaml::VpcId", stack - ) + stack_output_resolver = StackOutput("account/dev/vpc.yaml::VpcId", stack) stack_output_resolver.setup() assert stack.dependencies == ["account/dev/vpc.yaml"] @@ -44,16 +43,17 @@ def test_resolver(self, mock_get_output_value): result = stack_output_resolver.resolve() assert result == "output_value" mock_get_output_value.assert_called_once_with( - "meh-account-dev-vpc", "VpcId", - profile="dependency_profile", region="dependency_region", - iam_role="dependency_iam_role" + "meh-account-dev-vpc", + "VpcId", + profile="dependency_profile", + region="dependency_region", + sceptre_role="dependency_sceptre_role", ) - @patch( - "sceptre.resolvers.stack_output.StackOutput._get_output_value" - ) + @patch("sceptre.resolvers.stack_output.StackOutput._get_output_value") def test_resolver_with_existing_dependencies(self, mock_get_output_value): stack = MagicMock(spec=Stack) + stack.name = "my/stack" stack.dependencies = ["existing"] stack.project_code = "project-code" stack._connection_manager = MagicMock(spec=ConnectionManager) @@ -63,13 +63,11 @@ def test_resolver_with_existing_dependencies(self, mock_get_output_value): dependency.name = "account/dev/vpc" dependency.profile = "dependency_profile" dependency.region = "dependency_region" - dependency.iam_role = "dependency_iam_role" + dependency.sceptre_role = "dependency_sceptre_role" mock_get_output_value.return_value = "output_value" - stack_output_resolver = StackOutput( - "account/dev/vpc.yaml::VpcId", stack - ) + stack_output_resolver = StackOutput("account/dev/vpc.yaml::VpcId", stack) stack_output_resolver.setup() assert stack.dependencies == ["existing", "account/dev/vpc.yaml"] @@ -78,18 +76,17 @@ def test_resolver_with_existing_dependencies(self, mock_get_output_value): result = stack_output_resolver.resolve() assert result == "output_value" mock_get_output_value.assert_called_once_with( - "meh-account-dev-vpc", "VpcId", - profile="dependency_profile", region="dependency_region", - iam_role="dependency_iam_role" + "meh-account-dev-vpc", + "VpcId", + profile="dependency_profile", + region="dependency_region", + sceptre_role="dependency_sceptre_role", ) - @patch( - "sceptre.resolvers.stack_output.StackOutput._get_output_value" - ) - def test_resolve_with_implicit_stack_reference( - self, mock_get_output_value - ): + @patch("sceptre.resolvers.stack_output.StackOutput._get_output_value") + def test_resolve_with_implicit_stack_reference(self, mock_get_output_value): stack = MagicMock(spec=Stack) + stack.name = "my/stack" stack.dependencies = [] stack.project_code = "project-code" stack.name = "account/dev/stack" @@ -100,7 +97,7 @@ def test_resolve_with_implicit_stack_reference( dependency.name = "account/dev/vpc" dependency.profile = "dependency_profile" dependency.region = "dependency_region" - dependency.iam_role = "dependency_iam_role" + dependency.sceptre_role = "dependency_sceptre_role" mock_get_output_value.return_value = "output_value" @@ -113,18 +110,19 @@ def test_resolve_with_implicit_stack_reference( result = stack_output_resolver.resolve() assert result == "output_value" mock_get_output_value.assert_called_once_with( - "meh-account-dev-vpc", "VpcId", - profile="dependency_profile", region="dependency_region", - iam_role="dependency_iam_role" + "meh-account-dev-vpc", + "VpcId", + profile="dependency_profile", + region="dependency_region", + sceptre_role="dependency_sceptre_role", ) - @patch( - "sceptre.resolvers.stack_output.StackOutput._get_output_value" - ) + @patch("sceptre.resolvers.stack_output.StackOutput._get_output_value") def test_resolve_with_implicit_stack_reference_top_level( self, mock_get_output_value ): stack = MagicMock(spec=Stack) + stack.name = "my/stack" stack.dependencies = [] stack.project_code = "project-code" stack.name = "stack" @@ -135,7 +133,7 @@ def test_resolve_with_implicit_stack_reference_top_level( dependency.name = "vpc" dependency.profile = "dependency_profile" dependency.region = "dependency_region" - dependency.iam_role = "dependency_iam_role" + dependency.sceptre_role = "dependency_sceptre_role" mock_get_output_value.return_value = "output_value" @@ -148,19 +146,19 @@ def test_resolve_with_implicit_stack_reference_top_level( result = stack_output_resolver.resolve() assert result == "output_value" mock_get_output_value.assert_called_once_with( - "meh-vpc", "VpcId", - profile="dependency_profile", region="dependency_region", - iam_role="dependency_iam_role" + "meh-vpc", + "VpcId", + profile="dependency_profile", + region="dependency_region", + sceptre_role="dependency_sceptre_role", ) class TestStackOutputExternalResolver(object): - - @patch( - "sceptre.resolvers.stack_output.StackOutputExternal._get_output_value" - ) + @patch("sceptre.resolvers.stack_output.StackOutputExternal._get_output_value") def test_resolve(self, mock_get_output_value): stack = MagicMock(spec=Stack) + stack.name = "my/stack" stack.dependencies = [] stack._connection_manager = MagicMock(spec=ConnectionManager) stack_output_external_resolver = StackOutputExternal( @@ -173,20 +171,19 @@ def test_resolve(self, mock_get_output_value): ) assert stack.dependencies == [] - @patch( - "sceptre.resolvers.stack_output.StackOutputExternal._get_output_value" - ) + @patch("sceptre.resolvers.stack_output.StackOutputExternal._get_output_value") def test_resolve_with_args(self, mock_get_output_value): stack = MagicMock(spec=Stack) + stack.name = "my/stack" stack.dependencies = [] stack._connection_manager = MagicMock(spec=ConnectionManager) stack_output_external_resolver = StackOutputExternal( - "another/account-vpc::VpcId region::profile::iam_role", stack + "another/account-vpc::VpcId region::profile::sceptre_role", stack ) mock_get_output_value.return_value = "output_value" stack_output_external_resolver.resolve() mock_get_output_value.assert_called_once_with( - "another/account-vpc", "VpcId", "region", "profile", "iam_role" + "another/account-vpc", "VpcId", "region", "profile", "sceptre_role" ) assert stack.dependencies == [] @@ -207,19 +204,13 @@ def resolve(self): class TestStackOutputBaseResolver(object): - def setup_method(self, test_method): self.stack = MagicMock(spec=Stack) - self.stack._connection_manager = MagicMock( - spec=ConnectionManager - ) - self.base_stack_output_resolver = MockStackOutputBase( - None, self.stack - ) + self.stack.name = "my/stack.yaml" + self.stack._connection_manager = MagicMock(spec=ConnectionManager) + self.base_stack_output_resolver = MockStackOutputBase(None, self.stack) - @patch( - "sceptre.resolvers.stack_output.StackOutputBase._get_stack_outputs" - ) + @patch("sceptre.resolvers.stack_output.StackOutputBase._get_stack_outputs") def test_get_output_value_with_valid_key(self, mock_get_stack_outputs): mock_get_stack_outputs.return_value = {"key": "value"} @@ -229,9 +220,7 @@ def test_get_output_value_with_valid_key(self, mock_get_stack_outputs): assert response == "value" - @patch( - "sceptre.resolvers.stack_output.StackOutputBase._get_stack_outputs" - ) + @patch("sceptre.resolvers.stack_output.StackOutputBase._get_stack_outputs") def test_get_output_value_with_invalid_key(self, mock_get_stack_outputs): mock_get_stack_outputs.return_value = {"key": "value"} @@ -242,35 +231,32 @@ def test_get_output_value_with_invalid_key(self, mock_get_stack_outputs): def test_get_stack_outputs_with_valid_stack(self): self.stack.connection_manager.call.return_value = { - "Stacks": [{ - "Outputs": [ - { - "OutputKey": "key_1", - "OutputValue": "value_1", - "Description": "description_1" - }, - { - "OutputKey": "key_2", - "OutputValue": "value_2", - "Description": "description_2" - } - ] - }] + "Stacks": [ + { + "Outputs": [ + { + "OutputKey": "key_1", + "OutputValue": "value_1", + "Description": "description_1", + }, + { + "OutputKey": "key_2", + "OutputValue": "value_2", + "Description": "description_2", + }, + ] + } + ] } response = self.base_stack_output_resolver._get_stack_outputs( sentinel.stack_name ) - assert response == { - "key_1": "value_1", - "key_2": "value_2" - } + assert response == {"key_1": "value_1", "key_2": "value_2"} def test_get_stack_outputs_with_valid_stack_without_outputs(self): - self.stack.connection_manager.call.return_value = { - "Stacks": [{}] - } + self.stack.connection_manager.call.return_value = {"Stacks": [{}]} response = self.base_stack_output_resolver._get_stack_outputs( sentinel.stack_name @@ -279,32 +265,17 @@ def test_get_stack_outputs_with_valid_stack_without_outputs(self): def test_get_stack_outputs_with_unlaunched_stack(self): self.stack.connection_manager.call.side_effect = ClientError( - { - "Error": { - "Code": "404", - "Message": "stack does not exist" - } - }, - sentinel.operation + {"Error": {"Code": "404", "Message": "stack does not exist"}}, + sentinel.operation, ) with pytest.raises(StackDoesNotExistError): - self.base_stack_output_resolver._get_stack_outputs( - sentinel.stack_name - ) + self.base_stack_output_resolver._get_stack_outputs(sentinel.stack_name) def test_get_stack_outputs_with_unkown_boto_error(self): self.stack.connection_manager.call.side_effect = ClientError( - { - "Error": { - "Code": "500", - "Message": "Boom!" - } - }, - sentinel.operation + {"Error": {"Code": "500", "Message": "Boom!"}}, sentinel.operation ) with pytest.raises(ClientError): - self.base_stack_output_resolver._get_stack_outputs( - sentinel.stack_name - ) + self.base_stack_output_resolver._get_stack_outputs(sentinel.stack_name) diff --git a/tests/test_resolvers/test_sub.py b/tests/test_resolvers/test_sub.py new file mode 100644 index 000000000..7151f6eba --- /dev/null +++ b/tests/test_resolvers/test_sub.py @@ -0,0 +1,53 @@ +from unittest.mock import Mock + +import pytest + +from sceptre.exceptions import InvalidResolverArgumentError +from sceptre.resolvers import Resolver +from sceptre.resolvers.sub import Sub + + +class FirstResolver(Resolver): + def resolve(self): + return "first" + + +class SecondResolver(Resolver): + def resolve(self): + return "second" + + +class TestSub: + def test_resolve__combines_resolvers_into_single_string(self): + argument = [ + "{first} is {first_value}; {second} is {second_value}", + { + "first": FirstResolver(), + "second": SecondResolver(), + "first_value": 123, + "second_value": 456, + }, + ] + sub = Sub(argument, Mock()) + resolved = sub.resolve() + expected = "first is 123; second is 456" + assert expected == resolved + + @pytest.mark.parametrize( + "bad_argument", + [ + pytest.param("just a string", id="just a string"), + pytest.param([123, {"something": "else"}], id="first item is not string"), + pytest.param(["123", "hello"], id="second item is not a dict"), + pytest.param( + ["{this}", {"that": "hi"}], id="format string requires key not in dict" + ), + pytest.param(["first", ["second"], "third"], id="too many items"), + ], + ) + def test_resolve__invalid_arguments__raises_invalid_resolver_argument_error( + self, bad_argument + ): + sub = Sub(bad_argument, Mock()) + with pytest.raises(InvalidResolverArgumentError): + sub.resolve() diff --git a/tests/test_stack.py b/tests/test_stack.py index 9972a28c5..4683d0073 100644 --- a/tests/test_stack.py +++ b/tests/test_stack.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- -import importlib +from unittest.mock import MagicMock, sentinel -from mock import MagicMock, sentinel +import deprecation +import pytest + +from sceptre.exceptions import InvalidConfigFileError from sceptre.resolvers import Resolver from sceptre.stack import Stack from sceptre.template import Template @@ -10,63 +13,75 @@ def stack_factory(**kwargs): call_kwargs = { - 'name': 'dev/app/stack', - 'project_code': sentinel.project_code, - 'template_bucket_name': sentinel.template_bucket_name, - 'template_key_prefix': sentinel.template_key_prefix, - 'required_version': sentinel.required_version, - 'template_path': sentinel.template_path, - 'region': sentinel.region, - 'profile': sentinel.profile, - 'parameters': {"key1": "val1"}, - 'sceptre_user_data': sentinel.sceptre_user_data, - 'hooks': {}, - 's3_details': None, - 'dependencies': sentinel.dependencies, - 'role_arn': sentinel.role_arn, - 'protected': False, - 'tags': {"tag1": "val1"}, - 'external_name': sentinel.external_name, - 'notifications': [sentinel.notification], - 'on_failure': sentinel.on_failure, - 'stack_timeout': sentinel.stack_timeout, - 'stack_group_config': {} + "name": "dev/app/stack", + "project_code": sentinel.project_code, + "template_bucket_name": sentinel.template_bucket_name, + "template_key_prefix": sentinel.template_key_prefix, + "required_version": sentinel.required_version, + "template_path": sentinel.template_path, + "region": sentinel.region, + "profile": sentinel.profile, + "parameters": {"key1": "val1"}, + "sceptre_user_data": sentinel.sceptre_user_data, + "hooks": {}, + "s3_details": None, + "dependencies": sentinel.dependencies, + "role_arn": sentinel.role_arn, + "protected": False, + "tags": {"tag1": "val1"}, + "external_name": sentinel.external_name, + "notifications": [sentinel.notification], + "on_failure": sentinel.on_failure, + "disable_rollback": False, + "stack_timeout": sentinel.stack_timeout, + "stack_group_config": {}, } call_kwargs.update(kwargs) return Stack(**call_kwargs) class TestStack(object): - def setup_method(self, test_method): self.stack = Stack( - name='dev/app/stack', project_code=sentinel.project_code, + name="dev/app/stack", + project_code=sentinel.project_code, template_bucket_name=sentinel.template_bucket_name, template_key_prefix=sentinel.template_key_prefix, required_version=sentinel.required_version, - template_path=sentinel.template_path, region=sentinel.region, - profile=sentinel.profile, parameters={"key1": "val1"}, - sceptre_user_data=sentinel.sceptre_user_data, hooks={}, - s3_details=None, dependencies=sentinel.dependencies, - role_arn=sentinel.role_arn, protected=False, - tags={"tag1": "val1"}, external_name=sentinel.external_name, + template_path=sentinel.template_path, + region=sentinel.region, + profile=sentinel.profile, + parameters={"key1": "val1"}, + sceptre_user_data=sentinel.sceptre_user_data, + hooks={}, + s3_details=None, + dependencies=sentinel.dependencies, + role_arn=sentinel.role_arn, + protected=False, + tags={"tag1": "val1"}, + external_name=sentinel.external_name, notifications=[sentinel.notification], - on_failure=sentinel.on_failure, iam_role=sentinel.iam_role, + on_failure=sentinel.on_failure, + disable_rollback=False, + sceptre_role=sentinel.sceptre_role, + sceptre_role_session_duration=sentinel.sceptre_role_session_duration, stack_timeout=sentinel.stack_timeout, - stack_group_config={} + stack_group_config={}, ) self.stack._template = MagicMock(spec=Template) - def test_initiate_stack(self): + def test_initialize_stack_with_template_path(self): stack = Stack( - name='dev/stack/app', project_code=sentinel.project_code, + name="dev/stack/app", + project_code=sentinel.project_code, template_path=sentinel.template_path, template_bucket_name=sentinel.template_bucket_name, template_key_prefix=sentinel.template_key_prefix, required_version=sentinel.required_version, - region=sentinel.region, external_name=sentinel.external_name + region=sentinel.region, + external_name=sentinel.external_name, ) - assert stack.name == 'dev/stack/app' + assert stack.name == "dev/stack/app" assert stack.project_code == sentinel.project_code assert stack.template_bucket_name == sentinel.template_bucket_name assert stack.template_key_prefix == sentinel.template_key_prefix @@ -76,56 +91,197 @@ def test_initiate_stack(self): assert stack.parameters == {} assert stack.sceptre_user_data == {} assert stack.template_path == sentinel.template_path + assert stack.template_handler_config == { + "path": sentinel.template_path, + "type": "file", + } assert stack.s3_details is None assert stack._template is None assert stack.protected is False - assert stack.iam_role is None + assert stack.sceptre_role is None assert stack.role_arn is None assert stack.dependencies == [] assert stack.tags == {} assert stack.notifications == [] assert stack.on_failure is None + assert stack.disable_rollback is False assert stack.stack_group_config == {} + def test_initialize_stack_with_template_handler(self): + expected_template_handler_config = { + "type": "file", + "path": sentinel.template_path, + } + stack = Stack( + name="dev/stack/app", + project_code=sentinel.project_code, + template_handler_config=expected_template_handler_config, + template_bucket_name=sentinel.template_bucket_name, + template_key_prefix=sentinel.template_key_prefix, + required_version=sentinel.required_version, + region=sentinel.region, + external_name=sentinel.external_name, + ) + assert stack.name == "dev/stack/app" + assert stack.project_code == sentinel.project_code + assert stack.template_bucket_name == sentinel.template_bucket_name + assert stack.template_key_prefix == sentinel.template_key_prefix + assert stack.required_version == sentinel.required_version + assert stack.external_name == sentinel.external_name + assert stack.hooks == {} + assert stack.parameters == {} + assert stack.sceptre_user_data == {} + assert stack.template_path == sentinel.template_path + assert stack.template_handler_config == expected_template_handler_config + assert stack.s3_details is None + assert stack._template is None + assert stack.protected is False + assert stack.sceptre_role is None + assert stack.role_arn is None + assert stack.dependencies == [] + assert stack.tags == {} + assert stack.notifications == [] + assert stack.on_failure is None + assert stack.disable_rollback is False + assert stack.stack_group_config == {} + + def test_raises_exception_if_path_and_handler_configured(self): + with pytest.raises(InvalidConfigFileError): + Stack( + name="stack_name", + project_code="project_code", + template_path="template_path", + template_handler_config={"type": "file"}, + region="region", + ) + + def test_init__non_boolean_ignore_value__raises_invalid_config_file_error(self): + with pytest.raises(InvalidConfigFileError): + Stack( + name="dev/stack/app", + project_code=sentinel.project_code, + template_handler_config=sentinel.template_handler_config, + template_bucket_name=sentinel.template_bucket_name, + template_key_prefix=sentinel.template_key_prefix, + required_version=sentinel.required_version, + region=sentinel.region, + external_name=sentinel.external_name, + ignore="true", + ) + + def test_init__non_boolean_obsolete_value__raises_invalid_config_file_error(self): + with pytest.raises(InvalidConfigFileError): + Stack( + name="dev/stack/app", + project_code=sentinel.project_code, + template_handler_config=sentinel.template_handler_config, + template_bucket_name=sentinel.template_bucket_name, + template_key_prefix=sentinel.template_key_prefix, + required_version=sentinel.required_version, + region=sentinel.region, + external_name=sentinel.external_name, + obsolete="true", + ) + def test_stack_repr(self): - assert self.stack.__repr__() == \ - "sceptre.stack.Stack(" \ - "name='dev/app/stack', " \ - "project_code=sentinel.project_code, " \ - "template_path=sentinel.template_path, " \ - "region=sentinel.region, " \ - "template_bucket_name=sentinel.template_bucket_name, "\ - "template_key_prefix=sentinel.template_key_prefix, "\ - "required_version=sentinel.required_version, "\ - "iam_role=sentinel.iam_role, "\ - "profile=sentinel.profile, " \ - "sceptre_user_data=sentinel.sceptre_user_data, " \ - "parameters={'key1': 'val1'}, "\ - "hooks={}, "\ - "s3_details=None, " \ - "dependencies=sentinel.dependencies, "\ - "role_arn=sentinel.role_arn, "\ - "protected=False, "\ - "tags={'tag1': 'val1'}, "\ - "external_name=sentinel.external_name, " \ - "notifications=[sentinel.notification], " \ - "on_failure=sentinel.on_failure, " \ - "stack_timeout=sentinel.stack_timeout, " \ - "stack_group_config={}" \ + assert ( + self.stack.__repr__() == "sceptre.stack.Stack(" + "name='dev/app/stack', " + "project_code=sentinel.project_code, " + "template_handler_config={'type': 'file', 'path': sentinel.template_path}, " + "region=sentinel.region, " + "template_bucket_name=sentinel.template_bucket_name, " + "template_key_prefix=sentinel.template_key_prefix, " + "required_version=sentinel.required_version, " + "sceptre_role=sentinel.sceptre_role, " + "sceptre_role_session_duration=sentinel.sceptre_role_session_duration, " + "profile=sentinel.profile, " + "sceptre_user_data=sentinel.sceptre_user_data, " + "parameters={'key1': 'val1'}, " + "hooks={}, " + "s3_details=None, " + "dependencies=sentinel.dependencies, " + "cloudformation_service_role=sentinel.role_arn, " + "protected=False, " + "tags={'tag1': 'val1'}, " + "external_name=sentinel.external_name, " + "notifications=[sentinel.notification], " + "on_failure=sentinel.on_failure, " + "disable_rollback=False, " + "stack_timeout=sentinel.stack_timeout, " + "stack_group_config={}, " + "ignore=False, " + "obsolete=False" ")" - - def test_repr_can_eval_correctly(self): - sceptre = importlib.import_module('sceptre') - mock = importlib.import_module('mock') - evaluated_stack = eval( - repr(self.stack), - { - 'sceptre': sceptre, - 'sentinel': mock.mock.sentinel - } ) - assert isinstance(evaluated_stack, Stack) - assert evaluated_stack.__eq__(self.stack) + + def test_configuration_manager__sceptre_role_raises_recursive_resolve__returns_connection_manager_with_no_role( + self, + ): + class FakeResolver(Resolver): + def resolve(self): + return self.stack.sceptre_role + + self.stack.sceptre_role = FakeResolver() + + connection_manager = self.stack.connection_manager + assert connection_manager.sceptre_role is None + + def test_configuration_manager__sceptre_role_returns_value_second_access__returns_value_on_second_access( + self, + ): + class FakeResolver(Resolver): + access_count = 0 + + def resolve(self): + if self.access_count == 0: + self.access_count += 1 + return self.stack.sceptre_role + else: + return "role" + + self.stack.sceptre_role = FakeResolver() + + assert self.stack.connection_manager.sceptre_role is None + assert self.stack.connection_manager.sceptre_role == "role" + + def test_configuration_manager__sceptre_role_returns_value__returns_connection_manager_with_that_role( + self, + ): + class FakeResolver(Resolver): + def resolve(self): + return "role" + + self.stack.sceptre_role = FakeResolver() + + connection_manager = self.stack.connection_manager + assert connection_manager.sceptre_role == "role" + + @deprecation.fail_if_not_removed + def test_iam_role__is_removed_on_removal_version(self): + self.stack.iam_role + + @deprecation.fail_if_not_removed + def test_role_arn__is_removed_on_removal_version(self): + self.stack.role_arn + + @deprecation.fail_if_not_removed + def test_iam_role_session_duration__is_removed_on_removal_version(self): + self.stack.iam_role_session_duration + + def test_init__iam_role_set_resolves_to_sceptre_role(self): + stack = Stack("test", "test", "test", "test", iam_role="fancy") + assert stack.sceptre_role == "fancy" + + def test_init__role_arn_set_resolves_to_cloudformation_service_role(self): + stack = Stack("test", "test", "test", "test", role_arn="fancy") + assert stack.cloudformation_service_role == "fancy" + + def test_init__iam_role_session_duration_set_resolves_to_sceptre_role_session_duration( + self, + ): + stack = Stack("test", "test", "test", "test", iam_role_session_duration=123456) + assert stack.sceptre_role_session_duration == 123456 class TestStackSceptreUserData(object): @@ -134,8 +290,8 @@ def test_user_data_is_accessible(self): .sceptre_user_data is a property. Let's make sure it accesses the right data. """ - stack = stack_factory(sceptre_user_data={'test_key': sentinel.test_value}) - assert stack.sceptre_user_data['test_key'] is sentinel.test_value + stack = stack_factory(sceptre_user_data={"test_key": sentinel.test_value}) + assert stack.sceptre_user_data["test_key"] is sentinel.test_value def test_user_data_gets_resolved(self): class TestResolver(Resolver): @@ -145,24 +301,25 @@ def setup(self): def resolve(self): return sentinel.resolved_value - stack = stack_factory(sceptre_user_data={'test_key': TestResolver()}) - assert stack.sceptre_user_data['test_key'] is sentinel.resolved_value + stack = stack_factory(sceptre_user_data={"test_key": TestResolver()}) + assert stack.sceptre_user_data["test_key"] is sentinel.resolved_value def test_recursive_user_data_gets_resolved(self): """ .sceptre_user_data can have resolvers that refer to .sceptre_user_data itself. Those must be instantiated before the attribute can be used. """ + class TestResolver(Resolver): def setup(self): pass def resolve(self): - return self.stack.sceptre_user_data['primitive'] + return self.stack.sceptre_user_data["primitive"] stack = stack_factory() - stack._sceptre_user_data = { - 'primitive': sentinel.primitive_value, - 'resolved': TestResolver(stack=stack), + stack.sceptre_user_data = { + "primitive": sentinel.primitive_value, + "resolved": TestResolver(stack=stack), } - assert stack.sceptre_user_data['resolved'] == sentinel.primitive_value + assert stack.sceptre_user_data["resolved"] == sentinel.primitive_value diff --git a/tests/test_stack_status_colourer.py b/tests/test_stack_status_colourer.py index 3842f6b7a..802c1ee34 100644 --- a/tests/test_stack_status_colourer.py +++ b/tests/test_stack_status_colourer.py @@ -5,7 +5,6 @@ class TestStackStatusColourer(object): - def setup_method(self, test_method): init() self.stack_status_colourer = StackStatusColourer() @@ -26,7 +25,7 @@ def setup_method(self, test_method): "UPDATE_ROLLBACK_COMPLETE": Fore.GREEN, "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS": Fore.YELLOW, "UPDATE_ROLLBACK_FAILED": Fore.RED, - "UPDATE_ROLLBACK_IN_PROGRESS": Fore.YELLOW + "UPDATE_ROLLBACK_IN_PROGRESS": Fore.YELLOW, } def test_colour_with_string_with_no_stack_statuses(self): @@ -39,16 +38,11 @@ def test_colour_with_string_with_single_stack_status(self): for status in sorted(self.statuses.keys()) ] - responses = [ - self.stack_status_colourer.colour(string) - for string in strings - ] + responses = [self.stack_status_colourer.colour(string) for string in strings] assert responses == [ "string string {0}{1}{2} string".format( - self.statuses[status], - status, - Style.RESET_ALL + self.statuses[status], status, Style.RESET_ALL ) for status in sorted(self.statuses.keys()) ] @@ -57,11 +51,9 @@ def test_colour_with_string_with_multiple_stack_statuses(self): response = self.stack_status_colourer.colour( " ".join(sorted(self.statuses.keys())) ) - assert response == " ".join([ - "{0}{1}{2}".format( - self.statuses[status], - status, - Style.RESET_ALL - ) - for status in sorted(self.statuses.keys()) - ]) + assert response == " ".join( + [ + "{0}{1}{2}".format(self.statuses[status], status, Style.RESET_ALL) + for status in sorted(self.statuses.keys()) + ] + ) diff --git a/tests/test_template.py b/tests/test_template.py index e286275c2..fdfdb8480 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -6,20 +6,30 @@ import threading import pytest -from mock import patch, sentinel, Mock +from unittest.mock import patch, sentinel, Mock from freezegun import freeze_time from botocore.exceptions import ClientError -import sceptre.template from sceptre.template import Template from sceptre.connection_manager import ConnectionManager from sceptre.exceptions import UnsupportedTemplateFileTypeError -from sceptre.exceptions import TemplateSceptreHandlerError +from sceptre.exceptions import TemplateSceptreHandlerError, TemplateNotFoundError +from sceptre.template_handlers import TemplateHandler -class TestTemplate(object): +class MockTemplateHandler(TemplateHandler): + def __init__(self, *args, **kwargs): + super(MockTemplateHandler, self).__init__(*args, **kwargs) + + def schema(self): + return {} + + def handle(self): + return self.arguments["argument"] + +class TestTemplate(object): def setup_method(self, test_method): self.region = "region" self.bucket_name = "bucket_name" @@ -30,22 +40,43 @@ def setup_method(self, test_method): connection_manager.create_bucket_lock = threading.Lock() self.template = Template( - path="/folder/template.py", + name="template_name", + handler_config={"type": "file", "path": "/folder/template.py"}, sceptre_user_data={}, - connection_manager=connection_manager + stack_group_config={"project_path": "projects"}, + connection_manager=connection_manager, ) + def test_initialise_template_default_handler_type(self): + template = Template( + name="template_name", + handler_config={"path": "/folder/template.py"}, + sceptre_user_data={}, + stack_group_config={}, + connection_manager={}, + ) + + assert template.handler_config == { + "type": "file", + "path": "/folder/template.py", + } + def test_initialise_template(self): - assert self.template.path == "/folder/template.py" - assert self.template.name == "template" + assert self.template.handler_config == { + "type": "file", + "path": "/folder/template.py", + } + assert self.template.name == "template_name" assert self.template.sceptre_user_data == {} assert self.template._body is None def test_repr(self): representation = self.template.__repr__() - assert representation == "sceptre.template.Template(" \ - "name='template', path='/folder/template.py'"\ + assert ( + representation == "sceptre.template.Template(" + "name='template_name', handler_config={'type': 'file', 'path': '/folder/template.py'}" ", sceptre_user_data={}, s3_details=None)" + ) def test_body_with_cache(self): self.template._body = sentinel.body @@ -60,29 +91,41 @@ def test_upload_to_s3_with_valid_s3_details(self, mock_bucket_exists): self.template.s3_details = { "bucket_name": "bucket-name", "bucket_key": "bucket-key", - "bucket_region": "eu-west-1" } self.template.upload_to_s3() - self.template.connection_manager.call.assert_called_once_with( + ( + get_bucket_location_call, + put_object_call, + ) = self.template.connection_manager.call.call_args_list + get_bucket_location_call.assert_called_once_with( + service="s3", + command="get_bucket_location", + kwargs={"Bucket": "bucket-name"}, + ) + put_object_call.assert_called_once_with( service="s3", command="put_object", kwargs={ "Bucket": "bucket-name", "Key": "bucket-key", "Body": '{"template": "mock"}', - "ServerSideEncryption": "AES256" - } + "ServerSideEncryption": "AES256", + }, ) + def test_domain_from_region(self): + assert self.template._domain_from_region("us-east-1") == "com" + assert self.template._domain_from_region("cn-north-1") == "com.cn" + assert self.template._domain_from_region("cn-northwest-1") == "com.cn" + def test_bucket_exists_with_bucket_that_exists(self): # connection_manager.call doesn't raise an exception, mimicing the # behaviour when head_bucket successfully executes. self.template.s3_details = { "bucket_name": "bucket-name", "bucket_key": "bucket-key", - "bucket_region": "eu-west-1" } assert self.template._bucket_exists() is True @@ -92,18 +135,11 @@ def test_create_bucket_with_unreadable_bucket(self): self.template.s3_details = { "bucket_name": "bucket-name", "bucket_key": "bucket-key", - "bucket_region": "eu-west-1" } self.template.connection_manager.call.side_effect = ClientError( - { - "Error": { - "Code": 500, - "Message": "Bucket Unreadable" - } - }, - sentinel.operation - ) + {"Error": {"Code": 500, "Message": "Bucket Unreadable"}}, sentinel.operation + ) with pytest.raises(ClientError) as e: self.template._create_bucket() assert e.value.response["Error"]["Code"] == 500 @@ -115,20 +151,13 @@ def test_bucket_exists_with_non_existent_bucket(self): self.template.s3_details = { "bucket_name": "bucket-name", "bucket_key": "bucket-key", - "bucket_region": "eu-west-1" } self.template.connection_manager.call.side_effect = [ ClientError( - { - "Error": { - "Code": 404, - "Message": "Not Found" - } - }, - sentinel.operation + {"Error": {"Code": 404, "Message": "Not Found"}}, sentinel.operation ), - None + None, ] existance = self.template._bucket_exists() @@ -142,15 +171,12 @@ def test_create_bucket_in_us_east_1(self): self.template.s3_details = { "bucket_name": "bucket-name", "bucket_key": "bucket-key", - "bucket_region": "us-east-1" } self.template._create_bucket() self.template.connection_manager.call.assert_any_call( - service="s3", - command="create_bucket", - kwargs={"Bucket": "bucket-name"} + service="s3", command="create_bucket", kwargs={"Bucket": "bucket-name"} ) @patch("sceptre.template.Template.upload_to_s3") @@ -160,13 +186,25 @@ def test_get_boto_call_parameter_with_s3_details(self, mock_upload_to_s3): self.template.s3_details = { "bucket_name": sentinel.bucket_name, "bucket_key": sentinel.bucket_key, - "bucket_region": sentinel.bucket_region } boto_parameter = self.template.get_boto_call_parameter() assert boto_parameter == {"TemplateURL": sentinel.template_url} + def test_get_boto_call_parameter__has_s3_details_but_bucket_name_is_none__gets_template_body_dict( + self, + ): + self.template._body = sentinel.body + self.template.s3_details = { + "bucket_name": None, + "bucket_key": sentinel.bucket_key, + } + + boto_parameter = self.template.get_boto_call_parameter() + + assert boto_parameter == {"TemplateBody": sentinel.body} + def test_get_template_details_without_upload(self): self.template.s3_details = None self.template._body = sentinel.body @@ -176,21 +214,19 @@ def test_get_template_details_without_upload(self): def test_body_with_json_template(self): self.template.name = "vpc" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc.json" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures-vpc/templates/vpc.json" ) output = self.template.body - output_dict = json.loads(output) + output_dict = yaml.safe_load(output) with open("tests/fixtures/templates/compiled_vpc.json", "r") as f: expected_output_dict = json.loads(f.read()) assert output_dict == expected_output_dict def test_body_with_yaml_template(self): self.template.name = "vpc" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc.yaml" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc.yaml" ) output = self.template.body output_dict = yaml.safe_load(output) @@ -200,29 +236,41 @@ def test_body_with_yaml_template(self): def test_body_with_generic_template(self): self.template.name = "vpc" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc.template" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc.template" ) output = self.template.body - output_dict = json.loads(output) + output_dict = yaml.safe_load(output) with open("tests/fixtures/templates/compiled_vpc.json", "r") as f: expected_output_dict = json.loads(f.read()) assert output_dict == expected_output_dict + def test_body_with_chdir_template(self): + self.template.sceptre_user_data = None + self.template.name = "chdir" + current_dir = os.getcwd() + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/chdir.py" + ) + try: + yaml.safe_load(self.template.body) + except ValueError: + assert False + finally: + os.chdir(current_dir) + def test_body_with_missing_file(self): - self.template.path = "incorrect/template/path.py" - with pytest.raises(IOError): + self.template.handler_config["path"] = "incorrect/template/path.py" + with pytest.raises(TemplateNotFoundError): self.template.body def test_body_with_python_template(self): self.template.sceptre_user_data = None self.template.name = "vpc" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc.py" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc.py" ) - actual_output = json.loads(self.template.body) + actual_output = yaml.safe_load(self.template.body) with open("tests/fixtures/templates/compiled_vpc.json", "r") as f: expected_output = json.loads(f.read()) assert actual_output == expected_output @@ -230,117 +278,87 @@ def test_body_with_python_template(self): def test_body_with_python_template_with_sgt(self): self.template.sceptre_user_data = None self.template.name = "vpc_sgt" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc_sgt.py" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc_sgt.py" ) - actual_output = json.loads(self.template.body) + actual_output = yaml.safe_load(self.template.body) with open("tests/fixtures/templates/compiled_vpc.json", "r") as f: expected_output = json.loads(f.read()) assert actual_output == expected_output + def test_body_injects_yaml_start_marker(self): + self.template.name = "vpc" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc.without_start_marker.yaml" + ) + output = self.template.body + with open("tests/fixtures/templates/vpc.yaml", "r") as f: + expected_output = f.read() + assert output == expected_output + + def test_body_with_existing_yaml_start_marker(self): + self.template.name = "vpc" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc.yaml" + ) + output = self.template.body + with open("tests/fixtures/templates/vpc.yaml", "r") as f: + expected_output = f.read() + assert output == expected_output + + def test_body_with_existing_yaml_start_marker_j2(self): + self.template.name = "vpc" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc.yaml.j2" + ) + self.template.sceptre_user_data = {"vpc_id": "10.0.0.0/16"} + output = self.template.body + with open("tests/fixtures/templates/compiled_vpc.yaml", "r") as f: + expected_output = f.read() + assert output == expected_output.rstrip() + def test_body_injects_sceptre_user_data(self): - self.template.sceptre_user_data = { - "cidr_block": "10.0.0.0/16" - } + self.template.sceptre_user_data = {"cidr_block": "10.0.0.0/16"} self.template.name = "vpc_sud" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc_sud.py" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc_sud.py" ) - actual_output = json.loads(self.template.body) + actual_output = yaml.safe_load(self.template.body) with open("tests/fixtures/templates/compiled_vpc_sud.json", "r") as f: expected_output = json.loads(f.read()) assert actual_output == expected_output def test_body_injects_sceptre_user_data_incorrect_function(self): - self.template.sceptre_user_data = { - "cidr_block": "10.0.0.0/16" - } + self.template.sceptre_user_data = {"cidr_block": "10.0.0.0/16"} self.template.name = "vpc_sud_incorrect_function" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc_sud_incorrect_function.py" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc_sud_incorrect_function.py" ) with pytest.raises(TemplateSceptreHandlerError): self.template.body def test_body_injects_sceptre_user_data_incorrect_handler(self): - self.template.sceptre_user_data = { - "cidr_block": "10.0.0.0/16" - } + self.template.sceptre_user_data = {"cidr_block": "10.0.0.0/16"} self.template.name = "vpc_sud_incorrect_handler" - self.template.path = os.path.join( - os.getcwd(), - "tests/fixtures/templates/vpc_sud_incorrect_handler.py" + self.template.handler_config["path"] = os.path.join( + os.getcwd(), "tests/fixtures/templates/vpc_sud_incorrect_handler.py" ) with pytest.raises(TypeError): self.template.body def test_body_with_incorrect_filetype(self): - self.template.path = ( - "path/to/something.ext" - ) + self.template.handler_config["path"] = "path/to/something.ext" with pytest.raises(UnsupportedTemplateFileTypeError): self.template.body + def test_template_handler_is_called(self): + self.template.handler_config = { + "type": "test", + "argument": sentinel.template_handler_argument, + } + + self.template._registry = {"test": MockTemplateHandler} -@pytest.mark.parametrize("filename,sceptre_user_data,expected", [ - ( - "vpc.j2", - {"vpc_id": "10.0.0.0/16"}, - """Resources: - VPC: - Type: AWS::EC2::VPC - Properties: - CidrBlock: 10.0.0.0/16 -Outputs: - VpcId: - Value: - Ref: VPC""" - ), - ( - "vpc.yaml.j2", - {"vpc_id": "10.0.0.0/16"}, - """Resources: - VPC: - Type: AWS::EC2::VPC - Properties: - CidrBlock: 10.0.0.0/16 -Outputs: - VpcId: - Value: - Ref: VPC""" - ), - ( - "sg.j2", - [ - {"name": "sg_a", "inbound_ip": "10.0.0.0"}, - {"name": "sg_b", "inbound_ip": "10.0.0.1"} - ], - """Resources: - sg_a: - Type: "AWS::EC2::SecurityGroup" - Properties: - InboundIp: 10.0.0.0 - sg_b: - Type: "AWS::EC2::SecurityGroup" - Properties: - InboundIp: 10.0.0.1 -""" - ) -]) -def test_render_jinja_template(filename, sceptre_user_data, expected): - jinja_template_dir = os.path.join( - os.getcwd(), - "tests/fixtures/templates" - ) - result = sceptre.template.Template._render_jinja_template( - template_dir=jinja_template_dir, - filename=filename, - jinja_vars={"sceptre_user_data": sceptre_user_data} - ) - expected_yaml = yaml.safe_load(expected) - result_yaml = yaml.safe_load(result) - assert expected_yaml == result_yaml + result = self.template.body + assert result == "---\n" + str(sentinel.template_handler_argument) diff --git a/tests/test_template_handlers/test_file.py b/tests/test_template_handlers/test_file.py new file mode 100644 index 000000000..04b63c4c3 --- /dev/null +++ b/tests/test_template_handlers/test_file.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +import pytest + +from sceptre.template_handlers.file import File +from unittest.mock import patch, mock_open + + +class TestFile(object): + @pytest.mark.parametrize( + "project_path,path,output_path", + [ + ( + "my_project_dir", + "my.template.yaml", + "my_project_dir/templates/my.template.yaml", + ), # NOQA + ( + "/src/my_project_dir", + "my.template.yaml", + "/src/my_project_dir/templates/my.template.yaml", + ), # NOQA + ( + "my_project_dir", + "/src/my_project_dir/templates/my.template.yaml", + "/src/my_project_dir/templates/my.template.yaml", + ), # NOQA + ( + "/src/my_project_dir", + "/src/my_project_dir/templates/my.template.yaml", + "/src/my_project_dir/templates/my.template.yaml", + ), # NOQA + ], + ) + @patch("builtins.open", new_callable=mock_open, read_data="some_data") + def test_handler_open(self, mocked_open, project_path, path, output_path): + template_handler = File( + name="file_handler", + arguments={"path": path}, + stack_group_config={"project_path": project_path}, + ) + template_handler.handle() + mocked_open.assert_called_with(output_path) + + @pytest.mark.parametrize( + "project_path,path,output_path", + [ + ( + "my_project_dir", + "my.template.yaml.j2", + "my_project_dir/templates/my.template.yaml.j2", + ), # NOQA + ( + "/src/my_project_dir", + "my.template.yaml.j2", + "/src/my_project_dir/templates/my.template.yaml.j2", + ), # NOQA + ( + "my_project_dir", + "/src/my_project_dir/templates/my.template.yaml.j2", + "/src/my_project_dir/templates/my.template.yaml.j2", + ), # NOQA + ( + "/src/my_project_dir", + "/src/my_project_dir/templates/my.template.yaml.j2", + "/src/my_project_dir/templates/my.template.yaml.j2", + ), # NOQA + ], + ) + @patch("sceptre.template_handlers.helper.render_jinja_template") + def test_handler_render(self, mocked_render, project_path, path, output_path): + template_handler = File( + name="file_handler", + arguments={"path": path}, + stack_group_config={"project_path": project_path}, + ) + template_handler.handle() + mocked_render.assert_called_with(output_path, {"sceptre_user_data": None}, {}) + + @pytest.mark.parametrize( + "project_path,path,output_path", + [ + ( + "my_project_dir", + "my.template.yaml.py", + "my_project_dir/templates/my.template.yaml.py", + ), # NOQA + ( + "/src/my_project_dir", + "my.template.yaml.py", + "/src/my_project_dir/templates/my.template.yaml.py", + ), # NOQA + ( + "my_project_dir", + "/src/my_project_dir/templates/my.template.yaml.py", + "/src/my_project_dir/templates/my.template.yaml.py", + ), # NOQA + ( + "/src/my_project_dir", + "/src/my_project_dir/templates/my.template.yaml.py", + "/src/my_project_dir/templates/my.template.yaml.py", + ), # NOQA + ], + ) + @patch("sceptre.template_handlers.helper.call_sceptre_handler") + def test_handler_handler(self, mocked_handler, project_path, path, output_path): + template_handler = File( + name="file_handler", + arguments={"path": path}, + stack_group_config={"project_path": project_path}, + ) + template_handler.handle() + mocked_handler.assert_called_with(output_path, None) diff --git a/tests/test_template_handlers/test_helper.py b/tests/test_template_handlers/test_helper.py new file mode 100644 index 000000000..b798e8549 --- /dev/null +++ b/tests/test_template_handlers/test_helper.py @@ -0,0 +1,116 @@ +import os +import sys +import pytest +import yaml + +import sceptre.template_handlers.helper as helper +from sceptre.exceptions import TemplateNotFoundError +from unittest.mock import patch + + +@pytest.mark.parametrize( + "filename,sceptre_user_data,expected", + [ + ( + "vpc.j2", + {"vpc_id": "10.0.0.0/16"}, + """Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 +Outputs: + VpcId: + Value: + Ref: VPC""", + ), + ( + "vpc.yaml.j2", + {"vpc_id": "10.0.0.0/16"}, + """Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 +Outputs: + VpcId: + Value: + Ref: VPC""", + ), + ( + "sg.j2", + [ + {"name": "sg_a", "inbound_ip": "10.0.0.0"}, + {"name": "sg_b", "inbound_ip": "10.0.0.1"}, + ], + """Resources: + sg_a: + Type: "AWS::EC2::SecurityGroup" + Properties: + InboundIp: 10.0.0.0 + sg_b: + Type: "AWS::EC2::SecurityGroup" + Properties: + InboundIp: 10.0.0.1 +""", + ), + ], +) +@patch("pathlib.Path.exists") +def test_render_jinja_template(mock_pathlib, filename, sceptre_user_data, expected): + mock_pathlib.return_value = True + jinja_template_path = os.path.join( + os.getcwd(), "tests/fixtures/templates", filename + ) + result = helper.render_jinja_template( + path=jinja_template_path, + jinja_vars={"sceptre_user_data": sceptre_user_data}, + j2_environment={}, + ) + expected_yaml = yaml.safe_load(expected) + result_yaml = yaml.safe_load(result) + assert expected_yaml == result_yaml + + +@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires Python >= 3.8") +@pytest.mark.parametrize( + "j2_environment,expected_keys", + [ + ({}, ["autoescape", "loader", "undefined"]), + ( + {"lstrip_blocks": True}, + ["autoescape", "loader", "undefined", "lstrip_blocks"], + ), + ( + {"lstrip_blocks": True, "extensions": ["test-ext"]}, + ["autoescape", "loader", "undefined", "lstrip_blocks", "extensions"], + ), + ], +) +@patch("sceptre.template_handlers.helper.Environment") +@patch("pathlib.Path.exists") +def test_render_jinja_template_j2_environment_config( + mock_pathlib, mock_environment, j2_environment, expected_keys +): + mock_pathlib.return_value = True + filename = "vpc.j2" + sceptre_user_data = {"vpc_id": "10.0.0.0/16"} + jinja_template_path = os.path.join( + os.getcwd(), "tests/fixtures/templates", filename + ) + _ = helper.render_jinja_template( + path=jinja_template_path, + jinja_vars={"sceptre_user_data": sceptre_user_data}, + j2_environment=j2_environment, + ) + assert list(mock_environment.call_args.kwargs) == expected_keys + + +def test_render_jinja_template_non_existing_file(): + jinja_template_path = os.path.join("/ref/to/nowhere/boom.j2") + with pytest.raises(TemplateNotFoundError): + helper.render_jinja_template( + path=jinja_template_path, + jinja_vars={"sceptre_user_data": {}}, + j2_environment={}, + ) diff --git a/tests/test_template_handlers/test_http.py b/tests/test_template_handlers/test_http.py new file mode 100644 index 000000000..fe7fb4866 --- /dev/null +++ b/tests/test_template_handlers/test_http.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +import json +from unittest.mock import patch + +import pytest +from requests.exceptions import HTTPError + +from sceptre.exceptions import UnsupportedTemplateFileTypeError +from sceptre.template_handlers.http import Http + + +class TestHttp(object): + def test_get_template(self, requests_mock): + url = "https://raw.githubusercontent.com/acme/bucket.yaml" + requests_mock.get(url, content=b"Stuff is working") + template_handler = Http( + name="vpc", + arguments={"url": url}, + ) + result = template_handler.handle() + assert result == b"Stuff is working" + + def test_get_template__request_error__raises_error(self, requests_mock): + url = "https://raw.githubusercontent.com/acme/bucket.yaml" + requests_mock.get(url, content=b"Error message", status_code=404) + template_handler = Http( + name="vpc", + arguments={"url": url}, + ) + with pytest.raises(HTTPError): + template_handler.handle() + + def test_handler_unsupported_type(self): + handler = Http( + "http_handler", + {"url": "https://raw.githubusercontent.com/acme/bucket.unsupported"}, + ) + with pytest.raises(UnsupportedTemplateFileTypeError): + handler.handle() + + @pytest.mark.parametrize( + "url", + [ + ("https://raw.githubusercontent.com/acme/bucket.json"), + ("https://raw.githubusercontent.com/acme/bucket.yaml"), + ("https://raw.githubusercontent.com/acme/bucket.template"), + ], + ) + @patch("sceptre.template_handlers.http.Http._get_template") + def test_handler_raw_template(self, mock_get_template, url): + mock_get_template.return_value = {} + handler = Http("http_handler", {"url": url}) + handler.handle() + assert mock_get_template.call_count == 1 + + @patch("sceptre.template_handlers.helper.render_jinja_template") + @patch("sceptre.template_handlers.http.Http._get_template") + def test_handler_jinja_template( + self, mock_get_template, mock_render_jinja_template + ): + mock_get_template_response = { + "Description": "test template", + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "touchNothing": {"Type": "AWS::CloudFormation::WaitConditionHandle"} + }, + } + mock_get_template.return_value = json.dumps(mock_get_template_response).encode( + "utf-8" + ) + handler = Http( + "http_handler", {"url": "https://raw.githubusercontent.com/acme/bucket.j2"} + ) + handler.handle() + assert mock_render_jinja_template.call_count == 1 + + @patch("sceptre.template_handlers.helper.call_sceptre_handler") + @patch("sceptre.template_handlers.http.Http._get_template") + def test_handler_python_template( + self, mock_get_template, mock_call_sceptre_handler + ): + mock_get_template_response = { + "Description": "test template", + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "touchNothing": {"Type": "AWS::CloudFormation::WaitConditionHandle"} + }, + } + mock_get_template.return_value = json.dumps(mock_get_template_response).encode( + "utf-8" + ) + handler = Http( + "http_handler", {"url": "https://raw.githubusercontent.com/acme/bucket.py"} + ) + handler.handle() + assert mock_call_sceptre_handler.call_count == 1 + + @patch("sceptre.template_handlers.helper.call_sceptre_handler") + @patch("sceptre.template_handlers.http.Http._get_template") + def test_handler_override_handler_options( + self, mock_get_template, mock_call_sceptre_handler + ): + mock_get_template_response = { + "Description": "test template", + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "touchNothing": {"Type": "AWS::CloudFormation::WaitConditionHandle"} + }, + } + mock_get_template.return_value = json.dumps(mock_get_template_response).encode( + "utf-8" + ) + custom_handler_options = {"timeout": 10, "retries": 20} + handler = Http( + "http_handler", + {"url": "https://raw.githubusercontent.com/acme/bucket.py"}, + stack_group_config={"http_template_handler": custom_handler_options}, + ) + handler.handle() + assert mock_get_template.call_count == 1 + args, options = mock_get_template.call_args + assert options == {"timeout": 10, "retries": 20} diff --git a/tests/test_template_handlers/test_s3.py b/tests/test_template_handlers/test_s3.py new file mode 100644 index 000000000..8eaa77f01 --- /dev/null +++ b/tests/test_template_handlers/test_s3.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +import json +import io +import pytest + +from unittest.mock import MagicMock +from sceptre.connection_manager import ConnectionManager +from sceptre.exceptions import SceptreException, UnsupportedTemplateFileTypeError +from sceptre.template_handlers.s3 import S3 +from unittest.mock import patch + + +class TestS3(object): + def test_get_template(self): + connection_manager = MagicMock(spec=ConnectionManager) + connection_manager.call.return_value = {"Body": io.BytesIO(b"Stuff is working")} + template_handler = S3( + name="s3_handler", + arguments={"path": "bucket/folder/file.yaml"}, + connection_manager=connection_manager, + ) + result = template_handler.handle() + + connection_manager.call.assert_called_once_with( + service="s3", + command="get_object", + kwargs={"Bucket": "bucket", "Key": "folder/file.yaml"}, + ) + assert result == b"Stuff is working" + + def test_template_handler(self): + connection_manager = MagicMock(spec=ConnectionManager) + connection_manager.call.return_value = {"Body": io.BytesIO(b"Stuff is working")} + template_handler = S3( + name="vpc", + arguments={"path": "my-fancy-bucket/account/vpc.yaml"}, + connection_manager=connection_manager, + ) + result = template_handler.handle() + + connection_manager.call.assert_called_once_with( + service="s3", + command="get_object", + kwargs={"Bucket": "my-fancy-bucket", "Key": "account/vpc.yaml"}, + ) + assert result == b"Stuff is working" + + def test_invalid_response_reraises_exception(self): + connection_manager = MagicMock(spec=ConnectionManager) + connection_manager.call.side_effect = SceptreException("BOOM!") + + template_handler = S3( + name="vpc", + arguments={"path": "my-fancy-bucket/account/vpc.yaml"}, + connection_manager=connection_manager, + ) + + with pytest.raises(SceptreException) as e: + template_handler.handle() + + assert str(e.value) == "BOOM!" + + def test_handler_unsupported_type(self): + s3_handler = S3("s3_handler", {"path": "bucket/folder/file.unsupported"}) + with pytest.raises(UnsupportedTemplateFileTypeError): + s3_handler.handle() + + @pytest.mark.parametrize( + "path", + [ + ("bucket/folder/file.json"), + ("bucket/folder/file.yaml"), + ("bucket/folder/file.template"), + ], + ) + @patch("sceptre.template_handlers.s3.S3._get_template") + def test_handler_raw_template(self, mock_get_template, path): + mock_get_template.return_value = {} + s3_handler = S3("s3_handler", {"path": path}) + s3_handler.handle() + assert mock_get_template.call_count == 1 + + @patch("sceptre.template_handlers.helper.render_jinja_template") + @patch("sceptre.template_handlers.s3.S3._get_template") + def test_handler_jinja_template( + slef, mock_get_template, mock_render_jinja_template + ): + mock_get_template_response = { + "Description": "test template", + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "touchNothing": {"Type": "AWS::CloudFormation::WaitConditionHandle"} + }, + } + mock_get_template.return_value = json.dumps(mock_get_template_response).encode( + "utf-8" + ) + s3_handler = S3("s3_handler", {"path": "bucket/folder/file.j2"}) + s3_handler.handle() + assert mock_render_jinja_template.call_count == 1 + + @patch("sceptre.template_handlers.helper.call_sceptre_handler") + @patch("sceptre.template_handlers.s3.S3._get_template") + def test_handler_python_template( + self, mock_get_template, mock_call_sceptre_handler + ): + mock_get_template_response = { + "Description": "test template", + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "touchNothing": {"Type": "AWS::CloudFormation::WaitConditionHandle"} + }, + } + mock_get_template.return_value = json.dumps(mock_get_template_response).encode( + "utf-8" + ) + s3_handler = S3("s3_handler", {"path": "bucket/folder/file.py"}) + s3_handler.handle() + assert mock_call_sceptre_handler.call_count == 1 diff --git a/tests/test_template_handlers/test_template_handlers.py b/tests/test_template_handlers/test_template_handlers.py new file mode 100644 index 000000000..0ca885b63 --- /dev/null +++ b/tests/test_template_handlers/test_template_handlers.py @@ -0,0 +1,44 @@ +import logging +from unittest import TestCase + +import pytest + +from sceptre.exceptions import TemplateHandlerArgumentsInvalidError +from sceptre.template_handlers import TemplateHandler + + +class MockTemplateHandler(TemplateHandler): + def __init__(self, *args, **kwargs): + super(MockTemplateHandler, self).__init__(*args, **kwargs) + + def schema(self): + return { + "type": "object", + "properties": {"argument": {"type": "string"}}, + "required": ["argument"], + } + + def handle(self): + return "TestTemplateHandler" + + +class TestTemplateHandlers(TestCase): + def test_template_handler_validates_schema(self): + handler = MockTemplateHandler(name="mock", arguments={"argument": "test"}) + handler.validate() + + def test_template_handler_errors_when_arguments_invalid(self): + with pytest.raises(TemplateHandlerArgumentsInvalidError): + handler = MockTemplateHandler( + name="mock", arguments={"non-existent": "test"} + ) + handler.validate() + + def test_logger__logs_have_stack_name_prefix(self): + template_handler = MockTemplateHandler( + name="mock", arguments={"argument": "test"} + ) + with self.assertLogs(template_handler.logger.name, logging.INFO) as handler: + template_handler.logger.info("Bonjour") + + assert handler.records[0].message == f"{template_handler.name} - Bonjour" diff --git a/tox.ini b/tox.ini index 4da2d4b18..f437a2570 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,14 @@ [tox] -envlist = py27,py36,py37 +minversion = 3.23.0 +isolated_build = True +envlist = py{37,38,39,310},flake8 +skip_missing_interpreters = True [testenv] deps = -rrequirements/prod.txt -rrequirements/dev.txt -whitelist_externals = make -commands = make coverage +commands = pytest {posargs} -[pytest] -addopts = tests/ --ignore=env/ --ignore=venv/ --junitxml=test-results/junit.xml -s +[testenv:flake8] +commands = flake8 . diff --git a/venv.sh b/venv.sh new file mode 100755 index 000000000..6ccd7646c --- /dev/null +++ b/venv.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Also works with zsh. + +usage() { + echo "Usage: source $0" + echo "A script that sets up a Python Virtualenv" + exit 1 +} + +[ "$1" = "-h" ] && usage + +version="$(<.python-version)" +IFS="." read -r major minor _ <<< "$version" + +if ! python3 --version | grep -q "Python $major.$minor" ; then + echo "Please use pyenv and install Python $major.$minor" + return +fi + +virtualenv venv || \ + return + +. venv/bin/activate + +pip install -r requirements/prod.txt +pip install -r requirements/dev.txt +pip install -e .