From 963a3003eba0c8850fab86e323d846183d6e00e4 Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Mon, 17 Mar 2025 02:03:09 +0900 Subject: [PATCH 1/5] feat: Add if condition(Conditional Resource Evaluation) --- stackql_deploy/cmd/build.py | 16 +++++++++++++++- stackql_deploy/cmd/teardown.py | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/stackql_deploy/cmd/build.py b/stackql_deploy/cmd/build.py index 1851ce1..c126e78 100644 --- a/stackql_deploy/cmd/build.py +++ b/stackql_deploy/cmd/build.py @@ -5,7 +5,7 @@ run_ext_script, get_type ) -from ..lib.config import get_full_context +from ..lib.config import get_full_context, render_value from ..lib.templating import get_queries from .base import StackQLBase @@ -49,6 +49,20 @@ def run(self, dry_run, show_queries, on_failure): # get full context full_context = get_full_context(self.env, self.global_context, resource, self.logger) + # Check if the resource has an 'if' condition and evaluate it + if 'if' in resource: + condition = resource['if'] + try: + # Render the condition with the full context to resolve any template variables + rendered_condition = render_value(self.env, condition, full_context, self.logger) + # Evaluate the condition + condition_result = eval(rendered_condition) + if not condition_result: + self.logger.info(f"skipping resource [{resource['name']}] due to condition: {condition}") + continue + except Exception as e: + catch_error_and_exit(f"error evaluating condition for resource [{resource['name']}]: {e}", self.logger) + if type == 'script': self.process_script_resource(resource, dry_run, full_context) continue diff --git a/stackql_deploy/cmd/teardown.py b/stackql_deploy/cmd/teardown.py index 6aeda1c..ec3a7f4 100644 --- a/stackql_deploy/cmd/teardown.py +++ b/stackql_deploy/cmd/teardown.py @@ -3,7 +3,7 @@ catch_error_and_exit, get_type ) -from ..lib.config import get_full_context +from ..lib.config import get_full_context, render_value from ..lib.templating import get_queries from .base import StackQLBase @@ -63,6 +63,19 @@ def run(self, dry_run, show_queries, on_failure): # get full context full_context = get_full_context(self.env, self.global_context, resource, self.logger) + # Check if the resource has an 'if' condition and evaluate it + if 'if' in resource: + condition = resource['if'] + try: + # Render the condition with the full context to resolve any template variables + rendered_condition = render_value(self.env, condition, full_context, self.logger) + # Evaluate the condition + condition_result = eval(rendered_condition) + if not condition_result: + self.logger.info(f"skipping resource [{resource['name']}] due to condition: {condition}") + continue + except Exception as e: + catch_error_and_exit(f"error evaluating condition for resource [{resource['name']}]: {e}", self.logger) # add reverse export map variable to full context if 'exports' in resource: for export in resource['exports']: From c2a93aa5ddd384bb8dd1c39f41e4c777b622679f Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Mon, 17 Mar 2025 02:03:23 +0900 Subject: [PATCH 2/5] chore: Add docs --- website/docs/manifest-file.md | 8 +++- website/docs/manifest_fields/index.js | 47 +++++++++---------- website/docs/manifest_fields/resources.mdx | 3 +- website/docs/manifest_fields/resources/if.mdx | 47 +++++++++++++++++++ 4 files changed, 79 insertions(+), 26 deletions(-) create mode 100644 website/docs/manifest_fields/resources/if.mdx diff --git a/website/docs/manifest-file.md b/website/docs/manifest-file.md index 28fc31a..e787f6d 100644 --- a/website/docs/manifest-file.md +++ b/website/docs/manifest-file.md @@ -117,6 +117,12 @@ the fields within the __`stackql_manifest.yml`__ file are described in further d *** +### `resource.if` + + + +*** + ### `resource.props` @@ -390,4 +396,4 @@ resources: - {dest_range: "10.200.2.0/24", next_hop_ip: "10.240.0.22"} ``` - \ No newline at end of file + diff --git a/website/docs/manifest_fields/index.js b/website/docs/manifest_fields/index.js index 43f5937..43cc158 100644 --- a/website/docs/manifest_fields/index.js +++ b/website/docs/manifest_fields/index.js @@ -1,24 +1,23 @@ -export { default as Name } from './name.mdx'; -export { default as Description } from './description.mdx'; -export { default as Providers } from './providers.mdx'; -export { default as Globals } from './globals.mdx'; -export { default as GlobalName } from './globals/name.mdx'; -export { default as GlobalDescription } from './globals/description.mdx'; -export { default as GlobalValue } from './globals/value.mdx'; -export { default as Resources } from './resources.mdx'; -export { default as ResourceName } from './resources/name.mdx'; -export { default as ResourceType } from './resources/type.mdx'; -export { default as ResourceFile } from './resources/file.mdx'; -export { default as ResourceDescription } from './resources/description.mdx'; -export { default as ResourceExports } from './resources/exports.mdx'; -export { default as ResourceProps } from './resources/props.mdx'; -export { default as ResourceProtected } from './resources/protected.mdx'; -export { default as ResourceAuth } from './resources/auth.mdx'; -export { default as ResourcePropName } from './resources/props/name.mdx'; -export { default as ResourcePropDescription } from './resources/props/description.mdx'; -export { default as ResourcePropValue } from './resources/props/value.mdx'; -export { default as ResourcePropValues } from './resources/props/values.mdx'; -export { default as ResourcePropMerge } from './resources/props/merge.mdx'; -export { default as Version } from './version.mdx'; - - +export { default as Name } from "./name.mdx"; +export { default as Description } from "./description.mdx"; +export { default as Providers } from "./providers.mdx"; +export { default as Globals } from "./globals.mdx"; +export { default as GlobalName } from "./globals/name.mdx"; +export { default as GlobalDescription } from "./globals/description.mdx"; +export { default as GlobalValue } from "./globals/value.mdx"; +export { default as Resources } from "./resources.mdx"; +export { default as ResourceName } from "./resources/name.mdx"; +export { default as ResourceType } from "./resources/type.mdx"; +export { default as ResourceFile } from "./resources/file.mdx"; +export { default as ResourceDescription } from "./resources/description.mdx"; +export { default as ResourceExports } from "./resources/exports.mdx"; +export { default as ResourceProps } from "./resources/props.mdx"; +export { default as ResourceProtected } from "./resources/protected.mdx"; +export { default as ResourceAuth } from "./resources/auth.mdx"; +export { default as ResourceIf } from "./resources/if.mdx"; +export { default as ResourcePropName } from "./resources/props/name.mdx"; +export { default as ResourcePropDescription } from "./resources/props/description.mdx"; +export { default as ResourcePropValue } from "./resources/props/value.mdx"; +export { default as ResourcePropValues } from "./resources/props/values.mdx"; +export { default as ResourcePropMerge } from "./resources/props/merge.mdx"; +export { default as Version } from "./version.mdx"; diff --git a/website/docs/manifest_fields/resources.mdx b/website/docs/manifest_fields/resources.mdx index e2f64ef..9248bb0 100644 --- a/website/docs/manifest_fields/resources.mdx +++ b/website/docs/manifest_fields/resources.mdx @@ -12,6 +12,7 @@ import LeftAlignedTable from '@site/src/components/LeftAlignedTable'; { name: 'resource.exports', anchor: 'resourceexports' }, { name: 'resource.protected', anchor: 'resourceprotected' }, { name: 'resource.description', anchor: 'resourcedescription' }, + { name: 'resource.if', anchor: 'resourceif' }, ]} /> @@ -52,4 +53,4 @@ A file with the name of the resource with an `.iql` extension is expected to exi -::: \ No newline at end of file +::: diff --git a/website/docs/manifest_fields/resources/if.mdx b/website/docs/manifest_fields/resources/if.mdx new file mode 100644 index 0000000..e0c7de4 --- /dev/null +++ b/website/docs/manifest_fields/resources/if.mdx @@ -0,0 +1,47 @@ +import File from '@site/src/components/File'; +import LeftAlignedTable from '@site/src/components/LeftAlignedTable'; + + + +A conditional expression that determines whether a resource should be tested, provisioned, or deprovisioned. +You can use Python expressions to conditionally determine if a resource should be processed. + + + +```yaml {3} +resources: +- name: get_transfer_kms_key_id + if: "environment == 'production'" +... +``` + + + +:::info + +- Conditions are evaluated as Python expressions. +- You can reference literals (string, boolean, integer, etc.) or runtime template variables. +- If the condition evaluates to `True`, the resource is processed; if `False`, it is skipped. +- Template variables can be referenced using Jinja2 template syntax (`{{ variable }}`). + +::: + +## Examples + +Conditionally process a resource based on environment: + +```yaml +resources: + - name: get_transfer_kms_key_id + if: "environment == 'production'" + ... +``` + +Conditionally process based on other variable values: + +```yaml +resources: + - name: get_transfer_kms_key_id + if: "some_var == '{{ some_other_var_value }}'" + ... +``` From 80c87fb556c2704d4b0c207c72f02dd954bd39c3 Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Mon, 17 Mar 2025 02:03:41 +0900 Subject: [PATCH 3/5] chore: fix .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8c87ce0..ae1db4f 100644 --- a/.gitignore +++ b/.gitignore @@ -83,4 +83,7 @@ instance/ # Sphinx documentation docs/_build/ +.envrc + +venv/ .DS_Store From 94e3c44088721422bc40c0b748ae64a822c04c0d Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Mon, 17 Mar 2025 02:12:46 +0900 Subject: [PATCH 4/5] Add space --- stackql_deploy/cmd/teardown.py | 1 + test-derek-aws/README.md | 63 ++++++++++++++++++++++ test-derek-aws/resources/example_vpc.iql | 67 ++++++++++++++++++++++++ test-derek-aws/stackql_manifest.yml | 56 ++++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 test-derek-aws/README.md create mode 100644 test-derek-aws/resources/example_vpc.iql create mode 100644 test-derek-aws/stackql_manifest.yml diff --git a/stackql_deploy/cmd/teardown.py b/stackql_deploy/cmd/teardown.py index ec3a7f4..858f6eb 100644 --- a/stackql_deploy/cmd/teardown.py +++ b/stackql_deploy/cmd/teardown.py @@ -76,6 +76,7 @@ def run(self, dry_run, show_queries, on_failure): continue except Exception as e: catch_error_and_exit(f"error evaluating condition for resource [{resource['name']}]: {e}", self.logger) + # add reverse export map variable to full context if 'exports' in resource: for export in resource['exports']: diff --git a/test-derek-aws/README.md b/test-derek-aws/README.md new file mode 100644 index 0000000..3c89eb3 --- /dev/null +++ b/test-derek-aws/README.md @@ -0,0 +1,63 @@ +# `stackql-deploy` starter project for `aws` + +> for starter projects using other providers, try `stackql-deploy test-derek-aws --provider=azure` or `stackql-deploy test-derek-aws --provider=google` + +see the following links for more information on `stackql`, `stackql-deploy` and the `aws` provider: + +- [`aws` provider docs](https://stackql.io/registry/aws) +- [`stackql`](https://github.com/stackql/stackql) +- [`stackql-deploy` PyPI home page](https://pypi.org/project/stackql-deploy/) +- [`stackql-deploy` GitHub repo](https://github.com/stackql/stackql-deploy) + +## Overview + +__`stackql-deploy`__ is a stateless, declarative, SQL driven Infrastructure-as-Code (IaC) framework. There is no state file required as the current state is assessed for each resource at runtime. __`stackql-deploy`__ is capable of provisioning, deprovisioning and testing a stack which can include resources across different providers, like a stack spanning `aws` and `azure` for example. + +## Prerequisites + +This example requires `stackql-deploy` to be installed using __`pip install stackql-deploy`__. The host used to run `stackql-deploy` needs the necessary environment variables set to authenticate to your specific provider, in the case of the `aws` provider, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optionally `AWS_SESSION_TOKEN` must be set, for more information on authentication to `aws` see the [`aws` provider documentation](https://aws.stackql.io/providers/aws). + +## Usage + +Adjust the values in the [__`stackql_manifest.yml`__](stackql_manifest.yml) file if desired. The [__`stackql_manifest.yml`__](stackql_manifest.yml) file contains resource configuration variables to support multiple deployment environments, these will be used for `stackql` queries in the `resources` folder. + +The syntax for the `stackql-deploy` command is as follows: + +```bash +stackql-deploy { build | test | teardown } { stack-directory } { deployment environment} [ optional flags ] +``` + +### Deploying a stack + +For example, to deploy the stack named test-derek-aws to an environment labeled `sit`, run the following: + +```bash +stackql-deploy build test-derek-aws sit \ +-e AWS_REGION=ap-southeast-2 +``` + +Use the `--dry-run` flag to view the queries to be run without actually running them, for example: + +```bash +stackql-deploy build test-derek-aws sit \ +-e AWS_REGION=ap-southeast-2 \ +--dry-run +``` + +### Testing a stack + +To test a stack to ensure that all resources are present and in the desired state, run the following (in our `sit` deployment example): + +```bash +stackql-deploy test test-derek-aws sit \ +-e AWS_REGION=ap-southeast-2 +``` + +### Tearing down a stack + +To destroy or deprovision all resources in a stack for our `sit` deployment example, run the following: + +```bash +stackql-deploy teardown test-derek-aws sit \ +-e AWS_REGION=ap-southeast-2 +``` \ No newline at end of file diff --git a/test-derek-aws/resources/example_vpc.iql b/test-derek-aws/resources/example_vpc.iql new file mode 100644 index 0000000..463dbc1 --- /dev/null +++ b/test-derek-aws/resources/example_vpc.iql @@ -0,0 +1,67 @@ +/* defines the provisioning and deprovisioning commands +used to create, update or delete the resource +replace queries with your queries */ + +/*+ exists */ +SELECT COUNT(*) as count FROM +( +SELECT vpc_id, +json_group_object(tag_key, tag_value) as tags +FROM aws.ec2.vpc_tags +WHERE region = '{{ region }}' +AND cidr_block = '{{ vpc_cidr_block }}' +GROUP BY vpc_id +HAVING json_extract(tags, '$.Provisioner') = 'stackql' +AND json_extract(tags, '$.StackName') = '{{ stack_name }}' +AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' +) t; + +/*+ create */ +INSERT INTO aws.ec2.vpcs ( + CidrBlock, + Tags, + EnableDnsSupport, + EnableDnsHostnames, + region +) +SELECT + '{{ vpc_cidr_block }}', + '{{ vpc_tags }}', + true, + true, + '{{ region }}'; + +/*+ statecheck, retries=5, retry_delay=5 */ +SELECT COUNT(*) as count FROM +( +SELECT vpc_id, +cidr_block, +json_group_object(tag_key, tag_value) as tags +FROM aws.ec2.vpc_tags +WHERE region = '{{ region }}' +AND cidr_block = '{{ vpc_cidr_block }}' +GROUP BY vpc_id +HAVING json_extract(tags, '$.Provisioner') = 'stackql' +AND json_extract(tags, '$.StackName') = '{{ stack_name }}' +AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' +) t +WHERE cidr_block = '{{ vpc_cidr_block }}'; + +/*+ exports, retries=5, retry_delay=5 */ +SELECT vpc_id, vpc_cidr_block FROM +( +SELECT vpc_id, cidr_block as "vpc_cidr_block", +json_group_object(tag_key, tag_value) as tags +FROM aws.ec2.vpc_tags +WHERE region = '{{ region }}' +AND cidr_block = '{{ vpc_cidr_block }}' +GROUP BY vpc_id +HAVING json_extract(tags, '$.Provisioner') = 'stackql' +AND json_extract(tags, '$.StackName') = '{{ stack_name }}' +AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' +) t; + +/*+ delete */ +DELETE FROM aws.ec2.vpcs +WHERE data__Identifier = '{{ vpc_id }}' +AND region = '{{ region }}'; \ No newline at end of file diff --git a/test-derek-aws/stackql_manifest.yml b/test-derek-aws/stackql_manifest.yml new file mode 100644 index 0000000..c411627 --- /dev/null +++ b/test-derek-aws/stackql_manifest.yml @@ -0,0 +1,56 @@ +# +# aws starter project manifest file, add and update values as needed +# +version: 1 +name: "test-derek-aws" +description: description for "test-derek-aws" +providers: + - aws +globals: + - name: region + description: aws region + value: "{{ AWS_REGION }}" + - name: global_tags + value: + - Key: Provisioner + Value: stackql + - Key: StackName + Value: "{{ stack_name }}" + - Key: StackEnv + Value: "{{ stack_env }}" +resources: + - name: example_vpc + description: example vpc resource + if: "'{{ stack_env }}' == 'sit'" + props: + - name: vpc_cidr_block + values: + prd: + value: "10.0.0.0/16" + sit: + value: "10.1.0.0/16" + dev: + value: "10.2.0.0/16" + - name: vpc_tags + value: + - Key: Name + Value: "{{ stack_name }}-{{ stack_env }}-vpc" + merge: ['global_tags'] + exports: + - vpc_id + - vpc_cidr_block + - name: example_vpc_dev + description: example vpc resource for dev only + if: "'{{ stack_env }}' == 'dev'" + file: example_vpc.iql + props: + - name: vpc_cidr_block + value: "10.3.0.0/16" + - name: vpc_tags + value: + - Key: Name + Value: "{{ stack_name }}-{{ stack_env }}-vpc" + merge: ['global_tags'] + exports: + - vpc_id + - vpc_cidr_block From 4764f47134ee0d5e4e023f247205249a6478de7f Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Mon, 17 Mar 2025 02:16:08 +0900 Subject: [PATCH 5/5] fix lint error --- stackql_deploy/cmd/build.py | 5 ++++- stackql_deploy/cmd/teardown.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/stackql_deploy/cmd/build.py b/stackql_deploy/cmd/build.py index c126e78..8e3a4e6 100644 --- a/stackql_deploy/cmd/build.py +++ b/stackql_deploy/cmd/build.py @@ -61,7 +61,10 @@ def run(self, dry_run, show_queries, on_failure): self.logger.info(f"skipping resource [{resource['name']}] due to condition: {condition}") continue except Exception as e: - catch_error_and_exit(f"error evaluating condition for resource [{resource['name']}]: {e}", self.logger) + catch_error_and_exit( + f"error evaluating condition for resource [{resource['name']}]: {e}", + self.logger + ) if type == 'script': self.process_script_resource(resource, dry_run, full_context) diff --git a/stackql_deploy/cmd/teardown.py b/stackql_deploy/cmd/teardown.py index 858f6eb..17fd496 100644 --- a/stackql_deploy/cmd/teardown.py +++ b/stackql_deploy/cmd/teardown.py @@ -75,7 +75,10 @@ def run(self, dry_run, show_queries, on_failure): self.logger.info(f"skipping resource [{resource['name']}] due to condition: {condition}") continue except Exception as e: - catch_error_and_exit(f"error evaluating condition for resource [{resource['name']}]: {e}", self.logger) + catch_error_and_exit( + f"error evaluating condition for resource [{resource['name']}]: {e}", + self.logger + ) # add reverse export map variable to full context if 'exports' in resource: