From 72d22c867d6770ea14ac68da7f98cebbe26625cf Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Tue, 19 Nov 2024 07:18:52 -0500 Subject: [PATCH 01/37] update to github actions file to meet new demo structure --- .github/workflows/dev-pytest.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 94abf1e..bd20a59 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,16 +62,16 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | echo "::group::Setting up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/setup/${SCENARIO_NAME} + schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/1_setup echo "::endgroup::" echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/A__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/R__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/V1.0.0__render.sql + schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/2_test/A__render.sql + schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/2_test/R__render.sql + schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql echo "::endgroup::" echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" set +e - schemachange deploy --config-folder ./demo/${SCENARIO_NAME} + schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/2_test RESULT=$? if [ $RESULT -eq 0 ]; then echo "Deployment Completed!" @@ -81,7 +81,7 @@ jobs: echo "::endgroup::" set -e echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/teardown/${SCENARIO_NAME} + schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/3_teardown echo "::endgroup::" if [ $RESULT -ne 0 ]; then exit 1 From 7246545d79cb7cfccffbe56909f922bc87308acf Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 20 Nov 2024 17:21:09 -0500 Subject: [PATCH 02/37] Verifying Snowflake User --- .github/workflows/dev-pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index bd20a59..06275c6 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,6 +62,8 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | echo "::group::Setting up ${MY_TARGET_SCHEMA}" + echo "SNOWFLAKE_USER: ${SNOWFLAKE_USER}" + echo "SNOWFLAKE_ACCOUNT: ${SNOWFLAKE_ACCOUNT}" schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/1_setup echo "::endgroup::" echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" From c2bf720a6247a278eb27170744be426e6cca048e Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 21 Nov 2024 10:56:34 -0500 Subject: [PATCH 03/37] testing for demo functionality --- demo/basics_demo/schemachange-config.yml | 2 ++ schemachange/config/parse_cli_args.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/demo/basics_demo/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml index 7252e33..fab551b 100644 --- a/demo/basics_demo/schemachange-config.yml +++ b/demo/basics_demo/schemachange-config.yml @@ -1,6 +1,8 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" +snowflake-user: "{{ env_var(SNOWFLAKE_USER)}}" +snowflake-account: "{{ env_var(SNOWFLAKE_ACCOUNT)}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index 8b4cd01..4c19840 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -126,7 +126,7 @@ def parse_cli_args(args) -> dict: "-a", "--snowflake-account", type=str, - help="The name of the snowflake account (e.g. xy12345.east-us-2.azure)", + help="The name of the snowflake account (e.g. xy12345.east-us-2.azure, xy12345.east-us-2.azure.privatelink, org-accountname, org-accountname.privatelink)", required=False, action="deprecate", ) From c539c1bbc8adb560115a06e954acd7f2750316a3 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 21 Nov 2024 22:18:24 -0500 Subject: [PATCH 04/37] Fixing demo by making the config reference correct --- .github/workflows/dev-pytest.yml | 8 +++----- .../1_setup/schemachange-config.yml} | 2 +- .../2_test}/schemachange-config.yml | 2 +- demo/basics_demo/3_teardown/schemachange-config.yml | 11 +++++++++++ .../1_setup/schemachange-config.yml} | 2 +- .../2_test}/schemachange-config.yml | 4 +--- demo/citibike_demo/3_teardown/schemachange-config.yml | 11 +++++++++++ .../1_setup/schemachange-config.yml | 11 +++++++++++ .../{ => 2_test}/schemachange-config.yml | 2 +- .../3_teardown/schemachange-config.yml | 11 +++++++++++ 10 files changed, 52 insertions(+), 12 deletions(-) rename demo/{schemachange-config-setup.yml => basics_demo/1_setup/schemachange-config.yml} (87%) rename demo/{citibike_demo => basics_demo/2_test}/schemachange-config.yml (82%) create mode 100644 demo/basics_demo/3_teardown/schemachange-config.yml rename demo/{schemachange-config-teardown.yml => citibike_demo/1_setup/schemachange-config.yml} (86%) rename demo/{basics_demo => citibike_demo/2_test}/schemachange-config.yml (63%) create mode 100644 demo/citibike_demo/3_teardown/schemachange-config.yml create mode 100644 demo/citibike_demo_jinja/1_setup/schemachange-config.yml rename demo/citibike_demo_jinja/{ => 2_test}/schemachange-config.yml (90%) create mode 100644 demo/citibike_demo_jinja/3_teardown/schemachange-config.yml diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 06275c6..6dcda87 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,14 +62,12 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | echo "::group::Setting up ${MY_TARGET_SCHEMA}" - echo "SNOWFLAKE_USER: ${SNOWFLAKE_USER}" - echo "SNOWFLAKE_ACCOUNT: ${SNOWFLAKE_ACCOUNT}" schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/1_setup echo "::endgroup::" echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/2_test/A__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/2_test/R__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql + schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/A__render.sql + schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/R__render.sql + schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql echo "::endgroup::" echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" set +e diff --git a/demo/schemachange-config-setup.yml b/demo/basics_demo/1_setup/schemachange-config.yml similarity index 87% rename from demo/schemachange-config-setup.yml rename to demo/basics_demo/1_setup/schemachange-config.yml index e68543c..a407ad3 100644 --- a/demo/schemachange-config-setup.yml +++ b/demo/basics_demo/1_setup/schemachange-config.yml @@ -1,6 +1,6 @@ config-version: 1 -root-folder: "./demo/setup/{{ env_var('SCENARIO_NAME')}}" +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" diff --git a/demo/citibike_demo/schemachange-config.yml b/demo/basics_demo/2_test/schemachange-config.yml similarity index 82% rename from demo/citibike_demo/schemachange-config.yml rename to demo/basics_demo/2_test/schemachange-config.yml index 7252e33..932b7bc 100644 --- a/demo/citibike_demo/schemachange-config.yml +++ b/demo/basics_demo/2_test/schemachange-config.yml @@ -1,6 +1,6 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/basics_demo/3_teardown/schemachange-config.yml b/demo/basics_demo/3_teardown/schemachange-config.yml new file mode 100644 index 0000000..e31a396 --- /dev/null +++ b/demo/basics_demo/3_teardown/schemachange-config.yml @@ -0,0 +1,11 @@ +config-version: 1 + +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" + +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" + schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/schemachange-config-teardown.yml b/demo/citibike_demo/1_setup/schemachange-config.yml similarity index 86% rename from demo/schemachange-config-teardown.yml rename to demo/citibike_demo/1_setup/schemachange-config.yml index 800415a..a407ad3 100644 --- a/demo/schemachange-config-teardown.yml +++ b/demo/citibike_demo/1_setup/schemachange-config.yml @@ -1,6 +1,6 @@ config-version: 1 -root-folder: "./demo/teardown/{{ env_var('SCENARIO_NAME')}}" +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" diff --git a/demo/basics_demo/schemachange-config.yml b/demo/citibike_demo/2_test/schemachange-config.yml similarity index 63% rename from demo/basics_demo/schemachange-config.yml rename to demo/citibike_demo/2_test/schemachange-config.yml index fab551b..932b7bc 100644 --- a/demo/basics_demo/schemachange-config.yml +++ b/demo/citibike_demo/2_test/schemachange-config.yml @@ -1,8 +1,6 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" -snowflake-user: "{{ env_var(SNOWFLAKE_USER)}}" -snowflake-account: "{{ env_var(SNOWFLAKE_ACCOUNT)}}" +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/citibike_demo/3_teardown/schemachange-config.yml b/demo/citibike_demo/3_teardown/schemachange-config.yml new file mode 100644 index 0000000..e31a396 --- /dev/null +++ b/demo/citibike_demo/3_teardown/schemachange-config.yml @@ -0,0 +1,11 @@ +config-version: 1 + +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" + +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" + schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/citibike_demo_jinja/1_setup/schemachange-config.yml b/demo/citibike_demo_jinja/1_setup/schemachange-config.yml new file mode 100644 index 0000000..a407ad3 --- /dev/null +++ b/demo/citibike_demo_jinja/1_setup/schemachange-config.yml @@ -0,0 +1,11 @@ +config-version: 1 + +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" + +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" + schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/citibike_demo_jinja/schemachange-config.yml b/demo/citibike_demo_jinja/2_test/schemachange-config.yml similarity index 90% rename from demo/citibike_demo_jinja/schemachange-config.yml rename to demo/citibike_demo_jinja/2_test/schemachange-config.yml index 88cac3e..60f8e43 100644 --- a/demo/citibike_demo_jinja/schemachange-config.yml +++ b/demo/citibike_demo_jinja/2_test/schemachange-config.yml @@ -1,6 +1,6 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" modules-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/modules" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" diff --git a/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml b/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml new file mode 100644 index 0000000..e31a396 --- /dev/null +++ b/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml @@ -0,0 +1,11 @@ +config-version: 1 + +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" + +# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. +change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" +create-change-history-table: true + +vars: + database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" + schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" From 5ab793c44eb56f0b2b37a88d16add594af2fa30f Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 21 Nov 2024 22:25:48 -0500 Subject: [PATCH 05/37] testing basics demo --- demo/basics_demo/1_setup/schemachange-config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/demo/basics_demo/1_setup/schemachange-config.yml b/demo/basics_demo/1_setup/schemachange-config.yml index a407ad3..399967f 100644 --- a/demo/basics_demo/1_setup/schemachange-config.yml +++ b/demo/basics_demo/1_setup/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" From bca27a141aaa0ff243f3a2ce901b8b14badd076f Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 21 Nov 2024 22:28:49 -0500 Subject: [PATCH 06/37] setting up demo config --- demo/basics_demo/1_setup/schemachange-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/basics_demo/1_setup/schemachange-config.yml b/demo/basics_demo/1_setup/schemachange-config.yml index 399967f..a94ba6a 100644 --- a/demo/basics_demo/1_setup/schemachange-config.yml +++ b/demo/basics_demo/1_setup/schemachange-config.yml @@ -5,6 +5,7 @@ snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" +snowflake-password: "{{env_var('SNOWFLAKE_PASSWORD')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" From 2f6c220729ad71d35a93263fc10961d368072bde Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 08:07:31 -0500 Subject: [PATCH 07/37] testing demo scripts --- .github/workflows/dev-pytest.yml | 2 +- demo/basics_demo/1_setup/schemachange-config.yml | 1 - schemachange/config/get_merged_config.py | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 6dcda87..a8f2977 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja'] + scenario-name: [ 'basics_demo'] #, 'citibike_demo', 'citibike_demo_jinja'] os: ["ubuntu-latest"] python-version: ["3.12"] runs-on: ${{ matrix.os }} diff --git a/demo/basics_demo/1_setup/schemachange-config.yml b/demo/basics_demo/1_setup/schemachange-config.yml index a94ba6a..399967f 100644 --- a/demo/basics_demo/1_setup/schemachange-config.yml +++ b/demo/basics_demo/1_setup/schemachange-config.yml @@ -5,7 +5,6 @@ snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-password: "{{env_var('SNOWFLAKE_PASSWORD')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index f1a2ad5..bb1f870 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -12,6 +12,7 @@ load_yaml_config, validate_directory, validate_file_path, + get_snowflake_password, ) @@ -49,6 +50,7 @@ def get_merged_config( logger: structlog.BoundLogger, ) -> Union[DeployConfig, RenderConfig]: cli_kwargs = parse_cli_args(sys.argv[1:]) + cli_kwargs["password"] = get_snowflake_password() logger.debug("cli_kwargs", **cli_kwargs) cli_config_vars = cli_kwargs.pop("config_vars") From de1a3e278b75fad7846accdadafe2a057a3ed890 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 08:33:10 -0500 Subject: [PATCH 08/37] reverting the password reference --- schemachange/config/get_merged_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index bb1f870..604f1f4 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -11,8 +11,7 @@ from schemachange.config.utils import ( load_yaml_config, validate_directory, - validate_file_path, - get_snowflake_password, + validate_file_path ) @@ -50,7 +49,6 @@ def get_merged_config( logger: structlog.BoundLogger, ) -> Union[DeployConfig, RenderConfig]: cli_kwargs = parse_cli_args(sys.argv[1:]) - cli_kwargs["password"] = get_snowflake_password() logger.debug("cli_kwargs", **cli_kwargs) cli_config_vars = cli_kwargs.pop("config_vars") From 266a712677cdd24f88b965c22be42e51b1c44d1a Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 08:45:50 -0500 Subject: [PATCH 09/37] testing usage of snowflake password --- schemachange/config/DeployConfig.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index e183470..89abfe0 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -6,8 +6,10 @@ from schemachange.config.BaseConfig import BaseConfig from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.config.utils import get_snowflake_identifier_string - +from schemachange.config.utils import ( + get_snowflake_identifier_string, + get_snowflake_password, +) @dataclasses.dataclass(frozen=True) class DeployConfig(BaseConfig): @@ -82,6 +84,10 @@ def get_session_kwargs(self) -> dict: "connection_name": self.connection_name, "change_history_table": self.change_history_table, "autocommit": self.autocommit, - "query_tag": self.query_tag, + "query_tag": self.query_tag } + + snowflake_password = get_snowflake_password() + if snowflake_password is not None and snowflake_password: + session_kwargs["password"] = snowflake_password return {k: v for k, v in session_kwargs.items() if v is not None} From b9879b681bf9ce903856a08062c1eecc97282d9f Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 08:52:11 -0500 Subject: [PATCH 10/37] Setting right configs for demo --- demo/basics_demo/2_test/schemachange-config.yml | 4 ++++ demo/basics_demo/3_teardown/schemachange-config.yml | 4 ++++ demo/citibike_demo/2_test/schemachange-config.yml | 4 ++++ demo/citibike_demo_jinja/2_test/schemachange-config.yml | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/demo/basics_demo/2_test/schemachange-config.yml b/demo/basics_demo/2_test/schemachange-config.yml index 932b7bc..6ef4e6f 100644 --- a/demo/basics_demo/2_test/schemachange-config.yml +++ b/demo/basics_demo/2_test/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/basics_demo/3_teardown/schemachange-config.yml b/demo/basics_demo/3_teardown/schemachange-config.yml index e31a396..e3462f3 100644 --- a/demo/basics_demo/3_teardown/schemachange-config.yml +++ b/demo/basics_demo/3_teardown/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" diff --git a/demo/citibike_demo/2_test/schemachange-config.yml b/demo/citibike_demo/2_test/schemachange-config.yml index 932b7bc..6ef4e6f 100644 --- a/demo/citibike_demo/2_test/schemachange-config.yml +++ b/demo/citibike_demo/2_test/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/citibike_demo_jinja/2_test/schemachange-config.yml b/demo/citibike_demo_jinja/2_test/schemachange-config.yml index 60f8e43..3dab39f 100644 --- a/demo/citibike_demo_jinja/2_test/schemachange-config.yml +++ b/demo/citibike_demo_jinja/2_test/schemachange-config.yml @@ -2,6 +2,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" modules-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/modules" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true From 5b37fa41e881efde4d3edce37d5e0f77c64541df Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 09:13:57 -0500 Subject: [PATCH 11/37] Updating change history table creation logic --- schemachange/config/ChangeHistoryTable.py | 4 ++++ schemachange/session/SnowflakeSession.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/schemachange/config/ChangeHistoryTable.py b/schemachange/config/ChangeHistoryTable.py index ba7645a..7491911 100644 --- a/schemachange/config/ChangeHistoryTable.py +++ b/schemachange/config/ChangeHistoryTable.py @@ -17,6 +17,10 @@ class ChangeHistoryTable: @property def fully_qualified(self) -> str: return f"{self.database_name}.{self.schema_name}.{self.table_name}" + + @property + def fully_qualified_schema_name(self) -> str: + return f"{self.database_name}.{self.schema_name}" @classmethod def from_str(cls, table_str: str): diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index ba987f4..1346b76 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -149,7 +149,7 @@ def change_history_schema_exists(self) -> bool: return row[0] > 0 def create_change_history_schema(self, dry_run: bool) -> None: - query = f"CREATE SCHEMA IF NOT EXISTS {self.change_history_table.schema_name}" + query = f"CREATE SCHEMA IF NOT EXISTS {self.change_history_table.fully_qualified_schema_name}" if dry_run: self.logger.debug( "Running in dry-run mode. Skipping execution.", From dae6316a0045ee62e862e21b0d65850251a82af0 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 09:28:53 -0500 Subject: [PATCH 12/37] testing dev setup --- .github/workflows/dev-pytest.yml | 2 +- demo/citibike_demo/1_setup/schemachange-config.yml | 4 ++++ demo/citibike_demo/3_teardown/schemachange-config.yml | 4 ++++ demo/citibike_demo_jinja/1_setup/schemachange-config.yml | 4 ++++ demo/citibike_demo_jinja/3_teardown/schemachange-config.yml | 4 ++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index a8f2977..6dcda87 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - scenario-name: [ 'basics_demo'] #, 'citibike_demo', 'citibike_demo_jinja'] + scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja'] os: ["ubuntu-latest"] python-version: ["3.12"] runs-on: ${{ matrix.os }} diff --git a/demo/citibike_demo/1_setup/schemachange-config.yml b/demo/citibike_demo/1_setup/schemachange-config.yml index a407ad3..399967f 100644 --- a/demo/citibike_demo/1_setup/schemachange-config.yml +++ b/demo/citibike_demo/1_setup/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" diff --git a/demo/citibike_demo/3_teardown/schemachange-config.yml b/demo/citibike_demo/3_teardown/schemachange-config.yml index e31a396..e3462f3 100644 --- a/demo/citibike_demo/3_teardown/schemachange-config.yml +++ b/demo/citibike_demo/3_teardown/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" diff --git a/demo/citibike_demo_jinja/1_setup/schemachange-config.yml b/demo/citibike_demo_jinja/1_setup/schemachange-config.yml index a407ad3..399967f 100644 --- a/demo/citibike_demo_jinja/1_setup/schemachange-config.yml +++ b/demo/citibike_demo_jinja/1_setup/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" diff --git a/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml b/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml index e31a396..e3462f3 100644 --- a/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml +++ b/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml @@ -1,6 +1,10 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" +snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" +snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" +snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" +snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" From fd13a23b18317e0a790c7454466593015d514e93 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:04:05 -0500 Subject: [PATCH 13/37] testing github actions using shell script file --- .github/workflows/dev-pytest.yml | 29 ++--------------------------- testSchemchange.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 testSchemchange.sh diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 6dcda87..8153124 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja'] + scenario-name: [ 'basics_demo'] #, 'citibike_demo', 'citibike_demo_jinja'] os: ["ubuntu-latest"] python-version: ["3.12"] runs-on: ${{ matrix.os }} @@ -60,29 +60,4 @@ jobs: pytest # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema - run: | - echo "::group::Setting up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/1_setup - echo "::endgroup::" - echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/A__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/R__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql - echo "::endgroup::" - echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" - set +e - schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/2_test - RESULT=$? - if [ $RESULT -eq 0 ]; then - echo "Deployment Completed!" - else - echo "Deployment Failed. Proceeding to Teardown." - fi - echo "::endgroup::" - set -e - echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/3_teardown - echo "::endgroup::" - if [ $RESULT -ne 0 ]; then - exit 1 - fi + run: bash testSchemachange.sh \ No newline at end of file diff --git a/testSchemchange.sh b/testSchemchange.sh new file mode 100644 index 0000000..c5b76f4 --- /dev/null +++ b/testSchemchange.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository. +echo "::group::Setting up ${MY_TARGET_SCHEMA}" +schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/1_setup +echo "::endgroup::" + +echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" +schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/A__render.sql +schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/R__render.sql +schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql +echo "::endgroup::" +echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" + +set +e +schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/2_test +RESULT=$? + +if [ $RESULT -eq 0 ]; then + echo "Deployment Completed!" +else + echo "Deployment Failed. Proceeding to Teardown." +fi +echo "::endgroup::" +set -e + +echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" +schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/3_teardown +echo "::endgroup::" + +if [ $RESULT -ne 0 ]; then + exit 1 +fi \ No newline at end of file From b6e3e36966c5266d0b1f66d25ce43436e2b19437 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:19:37 -0500 Subject: [PATCH 14/37] tweaking scripts --- .github/workflows/dev-pytest.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 8153124..eb9f90e 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -60,4 +60,6 @@ jobs: pytest # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema - run: bash testSchemachange.sh \ No newline at end of file + run: | + chmod 700 testSchemchange + ./testSchemachange.sh \ No newline at end of file From db43352b15a2f4e6d535ef487ec1c260ed907a77 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:20:57 -0500 Subject: [PATCH 15/37] fix script --- .github/workflows/dev-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index eb9f90e..8de0724 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -61,5 +61,5 @@ jobs: # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | - chmod 700 testSchemchange + chmod 700 testSchemchange.sh ./testSchemachange.sh \ No newline at end of file From 59d52886398022a51fe47abe7ae94c9403d94a31 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:30:24 -0500 Subject: [PATCH 16/37] testing script --- .github/workflows/dev-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 8de0724..df7db44 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,4 +62,4 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | chmod 700 testSchemchange.sh - ./testSchemachange.sh \ No newline at end of file + bash testSchemachange.sh \ No newline at end of file From 04d2ab49509d2495f4e8e020c457157f54ab4f00 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:33:36 -0500 Subject: [PATCH 17/37] testing scripts --- .github/workflows/dev-pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index df7db44..c200bc2 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -58,8 +58,8 @@ jobs: id: pytest run: | pytest + chmod 700 testSchemchange.sh + pwd # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema - run: | - chmod 700 testSchemchange.sh - bash testSchemachange.sh \ No newline at end of file + run: testSchemachange.sh \ No newline at end of file From 3b659241b7ddd9fd875e890cb1fab96e2fad864f Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:42:06 -0500 Subject: [PATCH 18/37] testing scripts --- .github/workflows/dev-pytest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index c200bc2..5a5d2bd 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,4 +62,7 @@ jobs: pwd # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema - run: testSchemachange.sh \ No newline at end of file + run: | + pwd + ls -ltr + # ../testSchemachange.sh \ No newline at end of file From 02e20d89cb13a613a41143c50897ceed95a754ee Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:46:05 -0500 Subject: [PATCH 19/37] testing scripts --- .github/workflows/dev-pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 5a5d2bd..6fc9c7c 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -58,11 +58,11 @@ jobs: id: pytest run: | pytest - chmod 700 testSchemchange.sh + chmod 744 testSchemchange.sh pwd # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | pwd ls -ltr - # ../testSchemachange.sh \ No newline at end of file + ./testSchemachange.sh \ No newline at end of file From 4184af394444337a39ac27d696191edcff098861 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:48:15 -0500 Subject: [PATCH 20/37] testing scripts --- .github/workflows/dev-pytest.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 6fc9c7c..6531495 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -58,11 +58,9 @@ jobs: id: pytest run: | pytest - chmod 744 testSchemchange.sh - pwd # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | pwd - ls -ltr - ./testSchemachange.sh \ No newline at end of file + chmod 744 testSchemchange.sh + . ./testSchemachange.sh \ No newline at end of file From b3e84b88450bb8f250df86fd8aeeecbe7023144f Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 10:54:17 -0500 Subject: [PATCH 21/37] testing --- .github/workflows/dev-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 6531495..6374cd8 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -63,4 +63,4 @@ jobs: run: | pwd chmod 744 testSchemchange.sh - . ./testSchemachange.sh \ No newline at end of file + /home/runner/work/schemachange/schemachange/testSchemachange.sh \ No newline at end of file From 92186075c3b546c868fc5567d1dcacb3e0c7c79d Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 11:36:46 -0500 Subject: [PATCH 22/37] testing scripts --- .github/workflows/dev-pytest.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 6374cd8..a21c55d 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -3,6 +3,7 @@ name: dev-pytest defaults: run: shell: bash + working-directory: . on: workflow_dispatch: @@ -61,6 +62,7 @@ jobs: # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | - pwd + echo "Current Directory: ${PWD}" + ls -l chmod 744 testSchemchange.sh - /home/runner/work/schemachange/schemachange/testSchemachange.sh \ No newline at end of file + testSchemachange.sh \ No newline at end of file From adab2f5825566047a880ec60d9ca0b541a7cc7fa Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 11:53:53 -0500 Subject: [PATCH 23/37] setting working directory --- .github/workflows/dev-pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index a21c55d..1bcb711 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -3,7 +3,6 @@ name: dev-pytest defaults: run: shell: bash - working-directory: . on: workflow_dispatch: @@ -65,4 +64,5 @@ jobs: echo "Current Directory: ${PWD}" ls -l chmod 744 testSchemchange.sh - testSchemachange.sh \ No newline at end of file + testSchemachange.sh + working-directory: . \ No newline at end of file From 6aba92349e63669702e6e7cf5764869cb8693317 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 11:55:56 -0500 Subject: [PATCH 24/37] setting working directory --- .github/workflows/dev-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 1bcb711..9daa10b 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -65,4 +65,4 @@ jobs: ls -l chmod 744 testSchemchange.sh testSchemachange.sh - working-directory: . \ No newline at end of file + working-directory: $PWD \ No newline at end of file From 3fcdcbac2ecd92a3e420447b63423dec00c880a0 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 12:11:39 -0500 Subject: [PATCH 25/37] push script update --- .github/workflows/dev-pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 9daa10b..887262e 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -63,6 +63,6 @@ jobs: run: | echo "Current Directory: ${PWD}" ls -l - chmod 744 testSchemchange.sh - testSchemachange.sh - working-directory: $PWD \ No newline at end of file + chmod +x testSchemchange.sh + ./testSchemachange.sh + working-directory: . \ No newline at end of file From e482c3670e246f0d1bc28b8181218d8dc6c58db0 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 12:14:01 -0500 Subject: [PATCH 26/37] testing bash scripting --- .github/workflows/dev-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 887262e..479983f 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -64,5 +64,5 @@ jobs: echo "Current Directory: ${PWD}" ls -l chmod +x testSchemchange.sh - ./testSchemachange.sh + bash testSchemachange.sh working-directory: . \ No newline at end of file From c7d0a661fd226b904ecaa6924786ab8c21ace6b1 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 12:15:38 -0500 Subject: [PATCH 27/37] testing shell script --- .github/workflows/dev-pytest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 479983f..5d2d070 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -64,5 +64,4 @@ jobs: echo "Current Directory: ${PWD}" ls -l chmod +x testSchemchange.sh - bash testSchemachange.sh - working-directory: . \ No newline at end of file + more testSchemachange.sh \ No newline at end of file From 32b1c2c79ba9bfc3a2de390492e91215a2ebc941 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 12:34:26 -0500 Subject: [PATCH 28/37] it helps to use the right file name --- .github/workflows/dev-pytest.yml | 3 +-- testSchemchange.sh => testSchemachange.sh | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename testSchemchange.sh => testSchemachange.sh (100%) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 5d2d070..9972a29 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,6 +62,5 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | echo "Current Directory: ${PWD}" - ls -l chmod +x testSchemchange.sh - more testSchemachange.sh \ No newline at end of file + testSchemachange.sh \ No newline at end of file diff --git a/testSchemchange.sh b/testSchemachange.sh similarity index 100% rename from testSchemchange.sh rename to testSchemachange.sh From 9ab568bee3b6ffe87fb6cb881c4b5d9e484efbff Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 12:36:35 -0500 Subject: [PATCH 29/37] testing --- .github/workflows/dev-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 9972a29..419faad 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,5 +62,5 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | echo "Current Directory: ${PWD}" - chmod +x testSchemchange.sh + chmod +x testSchemachange.sh testSchemachange.sh \ No newline at end of file From e9b6918cb10f0d38127f4227ccdc70acd2073a29 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 12:40:04 -0500 Subject: [PATCH 30/37] testing continues --- .github/workflows/dev-pytest.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 419faad..61d3294 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -62,5 +62,7 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" chmod +x testSchemachange.sh - testSchemachange.sh \ No newline at end of file + bash testSchemachange.sh + working-directory: . \ No newline at end of file From 9c012f6f43358c881fa64bf368aa185c8f6c4dee Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 13:21:32 -0500 Subject: [PATCH 31/37] Testing scripting options --- .github/workflows/dev-pytest.yml | 7 +++ .github/workflows/master-pytest.yml | 73 ++++------------------------- populateConnection.sh | 12 +++++ testSchemachange.sh | 58 ++++++++++++++++++++++- testSchemachange2.sh | 57 ++++++++++++++++++++++ 5 files changed, 143 insertions(+), 64 deletions(-) create mode 100644 populateConnection.sh create mode 100644 testSchemachange2.sh diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 61d3294..1733851 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -54,6 +54,13 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Create and populate connections.toml + run: | + echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" + chmod +x populateConnection.sh + bash populateConnection.sh + working-directory: . - name: Test with pytest id: pytest run: | diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml index 027a09e..3fe0d00 100644 --- a/.github/workflows/master-pytest.yml +++ b/.github/workflows/master-pytest.yml @@ -60,16 +60,11 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Create and populate connections.toml run: | - touch ./connections.toml - echo [default] >> ./connections.toml - echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml - echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml - echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml - echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml - echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml - echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml - echo "cat connections.toml" - cat ./connections.toml + echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" + chmod +x populateConnection.sh + bash populateConnection.sh + working-directory: . - name: Test with pytest id: pytest run: | @@ -77,56 +72,8 @@ jobs: # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | - echo "::group::Setting up ${MY_TARGET_SCHEMA}" - schemachange deploy \ - --config-folder ./demo \ - --config-file-name schemachange-config-setup.yml \ - --root-folder ./demo/${SCENARIO_NAME}/1_setup \ - --connection-name default \ - --connections-file-path ./connections.toml \ - --verbose - echo "::endgroup::" - - echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - - schemachange render \ - --config-folder ./demo/${SCENARIO_NAME} \ - ./demo/${SCENARIO_NAME}/2_test/A__render.sql - schemachange render \ - --config-folder ./demo/${SCENARIO_NAME} \ - ./demo/${SCENARIO_NAME}/2_test/R__render.sql - schemachange render \ - --config-folder ./demo/${SCENARIO_NAME} \ - ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql - echo "::endgroup::" - - echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" - set +e - schemachange deploy \ - --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path ./connections.toml \ - --root-folder ./demo/${SCENARIO_NAME}/2_test \ - --verbose - RESULT=$? - if [ $RESULT -eq 0 ]; then - echo "Deployment Completed!" - else - echo "Deployment Failed. Proceeding to Teardown." - fi - echo "::endgroup::" - set -e - - echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" - schemachange deploy \ - --config-folder ./demo \ - --config-file-name schemachange-config-teardown.yml \ - --connection-name default \ - --connections-file-path ./connections.toml \ - --root-folder ./demo/${SCENARIO_NAME}/3_teardown \ - --verbose - echo "::endgroup::" - - if [ $RESULT -ne 0 ]; then - exit 1 - fi + echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" + chmod +x testSchemachange.sh + bash testSchemachange.sh + working-directory: . diff --git a/populateConnection.sh b/populateConnection.sh new file mode 100644 index 0000000..292039b --- /dev/null +++ b/populateConnection.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository. +touch ./connections.toml +echo [default] >> ./connections.toml +echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml +echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml +echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml +echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml +echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml +echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml +echo "cat connections.toml" +cat ./connections.toml \ No newline at end of file diff --git a/testSchemachange.sh b/testSchemachange.sh index c5b76f4..7c88483 100644 --- a/testSchemachange.sh +++ b/testSchemachange.sh @@ -29,4 +29,60 @@ echo "::endgroup::" if [ $RESULT -ne 0 ]; then exit 1 -fi \ No newline at end of file +fi + +echo "::group::Setting up ${MY_TARGET_SCHEMA}" +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-setup.yml \ +--root-folder ./demo/${SCENARIO_NAME}/1_setup \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--verbose +echo "::endgroup::" + +echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" + +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/A__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/R__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql +echo "::endgroup::" + +echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" +set +e +schemachange deploy \ +--config-folder ./demo/${SCENARIO_NAME} \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/2_test \ +--verbose +RESULT=$? + +if [ $RESULT -eq 0 ]; then + echo "Deployment Completed!" +else + echo "Deployment Failed. Proceeding to Teardown." +fi + echo "::endgroup::" + +set -e + +echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-teardown.yml \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/3_teardown \ +--verbose +echo "::endgroup::" + +if [ $RESULT -ne 0 ]; then + exit 1 +fi diff --git a/testSchemachange2.sh b/testSchemachange2.sh new file mode 100644 index 0000000..62e451d --- /dev/null +++ b/testSchemachange2.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository. +echo "::group::Setting up ${MY_TARGET_SCHEMA}" +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-setup.yml \ +--root-folder ./demo/${SCENARIO_NAME}/1_setup \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--verbose +echo "::endgroup::" + +echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" + +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/A__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/R__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql +echo "::endgroup::" + +echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" +set +e +schemachange deploy \ +--config-folder ./demo/${SCENARIO_NAME} \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/2_test \ +--verbose +RESULT=$? + +if [ $RESULT -eq 0 ]; then + echo "Deployment Completed!" +else + echo "Deployment Failed. Proceeding to Teardown." +fi + echo "::endgroup::" + +set -e + +echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-teardown.yml \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/3_teardown \ +--verbose +echo "::endgroup::" + +if [ $RESULT -ne 0 ]; then + exit 1 +fi From c3b53fab83fc4a8357fa00920b9d52b6dd5361e7 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 14:33:12 -0500 Subject: [PATCH 32/37] testing script to revert back to Zane's suggestion --- .github/workflows/dev-pytest.yml | 4 +- .../3_teardown/schemachange-config.yml | 15 ----- .../1_setup/schemachange-config.yml | 15 ----- .../3_teardown/schemachange-config.yml | 15 ----- .../{2_test => }/schemachange-config.yml | 6 -- .../3_teardown/schemachange-config.yml | 15 ----- .../{2_test => }/schemachange-config.yml | 5 -- ...nfig.yml => schemachange-config-setup.yml} | 6 -- ...g.yml => schemachange-config-teardown.yml} | 6 -- testSchemachange.sh | 58 +------------------ 10 files changed, 3 insertions(+), 142 deletions(-) delete mode 100644 demo/basics_demo/3_teardown/schemachange-config.yml delete mode 100644 demo/citibike_demo/1_setup/schemachange-config.yml delete mode 100644 demo/citibike_demo/3_teardown/schemachange-config.yml rename demo/citibike_demo/{2_test => }/schemachange-config.yml (50%) delete mode 100644 demo/citibike_demo_jinja/3_teardown/schemachange-config.yml rename demo/citibike_demo_jinja/{2_test => }/schemachange-config.yml (67%) rename demo/{basics_demo/1_setup/schemachange-config.yml => schemachange-config-setup.yml} (60%) rename demo/{citibike_demo_jinja/1_setup/schemachange-config.yml => schemachange-config-teardown.yml} (60%) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 1733851..9438669 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -70,6 +70,6 @@ jobs: run: | echo "Current Directory: ${PWD}" echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" - chmod +x testSchemachange.sh - bash testSchemachange.sh + chmod +x testSchemachange2.sh + bash testSchemachange2.sh working-directory: . \ No newline at end of file diff --git a/demo/basics_demo/3_teardown/schemachange-config.yml b/demo/basics_demo/3_teardown/schemachange-config.yml deleted file mode 100644 index e3462f3..0000000 --- a/demo/basics_demo/3_teardown/schemachange-config.yml +++ /dev/null @@ -1,15 +0,0 @@ -config-version: 1 - -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" - -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/citibike_demo/1_setup/schemachange-config.yml b/demo/citibike_demo/1_setup/schemachange-config.yml deleted file mode 100644 index 399967f..0000000 --- a/demo/citibike_demo/1_setup/schemachange-config.yml +++ /dev/null @@ -1,15 +0,0 @@ -config-version: 1 - -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" - -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/citibike_demo/3_teardown/schemachange-config.yml b/demo/citibike_demo/3_teardown/schemachange-config.yml deleted file mode 100644 index e3462f3..0000000 --- a/demo/citibike_demo/3_teardown/schemachange-config.yml +++ /dev/null @@ -1,15 +0,0 @@ -config-version: 1 - -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" - -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/citibike_demo/2_test/schemachange-config.yml b/demo/citibike_demo/schemachange-config.yml similarity index 50% rename from demo/citibike_demo/2_test/schemachange-config.yml rename to demo/citibike_demo/schemachange-config.yml index 6ef4e6f..da58e63 100644 --- a/demo/citibike_demo/2_test/schemachange-config.yml +++ b/demo/citibike_demo/schemachange-config.yml @@ -1,11 +1,5 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" - change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml b/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml deleted file mode 100644 index e3462f3..0000000 --- a/demo/citibike_demo_jinja/3_teardown/schemachange-config.yml +++ /dev/null @@ -1,15 +0,0 @@ -config-version: 1 - -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/3_teardown" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" - -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/citibike_demo_jinja/2_test/schemachange-config.yml b/demo/citibike_demo_jinja/schemachange-config.yml similarity index 67% rename from demo/citibike_demo_jinja/2_test/schemachange-config.yml rename to demo/citibike_demo_jinja/schemachange-config.yml index 3dab39f..e80ca9b 100644 --- a/demo/citibike_demo_jinja/2_test/schemachange-config.yml +++ b/demo/citibike_demo_jinja/schemachange-config.yml @@ -1,11 +1,6 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" modules-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/modules" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/basics_demo/1_setup/schemachange-config.yml b/demo/schemachange-config-setup.yml similarity index 60% rename from demo/basics_demo/1_setup/schemachange-config.yml rename to demo/schemachange-config-setup.yml index 399967f..906978d 100644 --- a/demo/basics_demo/1_setup/schemachange-config.yml +++ b/demo/schemachange-config-setup.yml @@ -1,11 +1,5 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" - # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/citibike_demo_jinja/1_setup/schemachange-config.yml b/demo/schemachange-config-teardown.yml similarity index 60% rename from demo/citibike_demo_jinja/1_setup/schemachange-config.yml rename to demo/schemachange-config-teardown.yml index 399967f..906978d 100644 --- a/demo/citibike_demo_jinja/1_setup/schemachange-config.yml +++ b/demo/schemachange-config-teardown.yml @@ -1,11 +1,5 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/1_setup" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" - # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" create-change-history-table: true diff --git a/testSchemachange.sh b/testSchemachange.sh index 7c88483..c5b76f4 100644 --- a/testSchemachange.sh +++ b/testSchemachange.sh @@ -29,60 +29,4 @@ echo "::endgroup::" if [ $RESULT -ne 0 ]; then exit 1 -fi - -echo "::group::Setting up ${MY_TARGET_SCHEMA}" -schemachange deploy \ ---config-folder ./demo \ ---config-file-name schemachange-config-setup.yml \ ---root-folder ./demo/${SCENARIO_NAME}/1_setup \ ---connection-name default \ ---connections-file-path ./connections.toml \ ---verbose -echo "::endgroup::" - -echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - -schemachange render \ ---config-folder ./demo/${SCENARIO_NAME} \ -./demo/${SCENARIO_NAME}/2_test/A__render.sql -schemachange render \ ---config-folder ./demo/${SCENARIO_NAME} \ -./demo/${SCENARIO_NAME}/2_test/R__render.sql -schemachange render \ ---config-folder ./demo/${SCENARIO_NAME} \ -./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql -echo "::endgroup::" - -echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" -set +e -schemachange deploy \ ---config-folder ./demo/${SCENARIO_NAME} \ ---connection-name default \ ---connections-file-path ./connections.toml \ ---root-folder ./demo/${SCENARIO_NAME}/2_test \ ---verbose -RESULT=$? - -if [ $RESULT -eq 0 ]; then - echo "Deployment Completed!" -else - echo "Deployment Failed. Proceeding to Teardown." -fi - echo "::endgroup::" - -set -e - -echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" -schemachange deploy \ ---config-folder ./demo \ ---config-file-name schemachange-config-teardown.yml \ ---connection-name default \ ---connections-file-path ./connections.toml \ ---root-folder ./demo/${SCENARIO_NAME}/3_teardown \ ---verbose -echo "::endgroup::" - -if [ $RESULT -ne 0 ]; then - exit 1 -fi +fi \ No newline at end of file From 0f73900f610b157bc0770fe25dd3cfd23263833d Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 14:38:26 -0500 Subject: [PATCH 33/37] update to schemachange config location --- demo/basics_demo/{2_test => }/schemachange-config.yml | 4 ---- 1 file changed, 4 deletions(-) rename demo/basics_demo/{2_test => }/schemachange-config.yml (61%) diff --git a/demo/basics_demo/2_test/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml similarity index 61% rename from demo/basics_demo/2_test/schemachange-config.yml rename to demo/basics_demo/schemachange-config.yml index 6ef4e6f..932b7bc 100644 --- a/demo/basics_demo/2_test/schemachange-config.yml +++ b/demo/basics_demo/schemachange-config.yml @@ -1,10 +1,6 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" -snowflake-user: "{{env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{env_var('SNOWFLAKE_WAREHOUSE')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true From 62e947811819444a70981d280218d00fbf33326f Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 14:44:32 -0500 Subject: [PATCH 34/37] Pushing pre-final changes --- .github/workflows/dev-pytest.yml | 4 +-- testSchemachange.sh | 43 +++++++++++++++++++----- testSchemachange2.sh | 57 -------------------------------- 3 files changed, 36 insertions(+), 68 deletions(-) delete mode 100644 testSchemachange2.sh diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 9438669..1733851 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -70,6 +70,6 @@ jobs: run: | echo "Current Directory: ${PWD}" echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" - chmod +x testSchemachange2.sh - bash testSchemachange2.sh + chmod +x testSchemachange.sh + bash testSchemachange.sh working-directory: . \ No newline at end of file diff --git a/testSchemachange.sh b/testSchemachange.sh index c5b76f4..62e451d 100644 --- a/testSchemachange.sh +++ b/testSchemachange.sh @@ -1,18 +1,36 @@ #!/bin/bash # Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository. echo "::group::Setting up ${MY_TARGET_SCHEMA}" -schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/1_setup +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-setup.yml \ +--root-folder ./demo/${SCENARIO_NAME}/1_setup \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--verbose echo "::endgroup::" echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" -schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/A__render.sql -schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/R__render.sql -schemachange render --config-folder ./demo/${SCENARIO_NAME}/2_test ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql + +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/A__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/R__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql echo "::endgroup::" -echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" +echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" set +e -schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/2_test +schemachange deploy \ +--config-folder ./demo/${SCENARIO_NAME} \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/2_test \ +--verbose RESULT=$? if [ $RESULT -eq 0 ]; then @@ -20,13 +38,20 @@ if [ $RESULT -eq 0 ]; then else echo "Deployment Failed. Proceeding to Teardown." fi -echo "::endgroup::" + echo "::endgroup::" + set -e echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" -schemachange deploy --config-folder ./demo/${SCENARIO_NAME}/3_teardown +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-teardown.yml \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/3_teardown \ +--verbose echo "::endgroup::" if [ $RESULT -ne 0 ]; then exit 1 -fi \ No newline at end of file +fi diff --git a/testSchemachange2.sh b/testSchemachange2.sh deleted file mode 100644 index 62e451d..0000000 --- a/testSchemachange2.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository. -echo "::group::Setting up ${MY_TARGET_SCHEMA}" -schemachange deploy \ ---config-folder ./demo \ ---config-file-name schemachange-config-setup.yml \ ---root-folder ./demo/${SCENARIO_NAME}/1_setup \ ---connection-name default \ ---connections-file-path ./connections.toml \ ---verbose -echo "::endgroup::" - -echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - -schemachange render \ ---config-folder ./demo/${SCENARIO_NAME} \ -./demo/${SCENARIO_NAME}/2_test/A__render.sql -schemachange render \ ---config-folder ./demo/${SCENARIO_NAME} \ -./demo/${SCENARIO_NAME}/2_test/R__render.sql -schemachange render \ ---config-folder ./demo/${SCENARIO_NAME} \ -./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql -echo "::endgroup::" - -echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" -set +e -schemachange deploy \ ---config-folder ./demo/${SCENARIO_NAME} \ ---connection-name default \ ---connections-file-path ./connections.toml \ ---root-folder ./demo/${SCENARIO_NAME}/2_test \ ---verbose -RESULT=$? - -if [ $RESULT -eq 0 ]; then - echo "Deployment Completed!" -else - echo "Deployment Failed. Proceeding to Teardown." -fi - echo "::endgroup::" - -set -e - -echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" -schemachange deploy \ ---config-folder ./demo \ ---config-file-name schemachange-config-teardown.yml \ ---connection-name default \ ---connections-file-path ./connections.toml \ ---root-folder ./demo/${SCENARIO_NAME}/3_teardown \ ---verbose -echo "::endgroup::" - -if [ $RESULT -ne 0 ]; then - exit 1 -fi From e3a02b353fc2e0b4984c359d7bf8f9fa0b42c781 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 14:45:06 -0500 Subject: [PATCH 35/37] adding test scenarios in Dev --- .github/workflows/dev-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 1733851..e325e82 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - scenario-name: [ 'basics_demo'] #, 'citibike_demo', 'citibike_demo_jinja'] + scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja'] os: ["ubuntu-latest"] python-version: ["3.12"] runs-on: ${{ matrix.os }} From 7e68ecfd6d56bca1bc4a9dadb90dbc20c9c9b0c9 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 14:49:14 -0500 Subject: [PATCH 36/37] testing Deployconfig changes --- schemachange/config/DeployConfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 89abfe0..295839a 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -87,7 +87,7 @@ def get_session_kwargs(self) -> dict: "query_tag": self.query_tag } - snowflake_password = get_snowflake_password() - if snowflake_password is not None and snowflake_password: - session_kwargs["password"] = snowflake_password + #snowflake_password = get_snowflake_password() + #if snowflake_password is not None and snowflake_password: + # session_kwargs["password"] = snowflake_password return {k: v for k, v in session_kwargs.items() if v is not None} From c05fa569d6fe229848e271d46ae968e1a9b74a44 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 22 Nov 2024 14:55:38 -0500 Subject: [PATCH 37/37] Leaving code comment to review later --- schemachange/config/DeployConfig.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 295839a..b42fc52 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -87,7 +87,9 @@ def get_session_kwargs(self) -> dict: "query_tag": self.query_tag } - #snowflake_password = get_snowflake_password() - #if snowflake_password is not None and snowflake_password: + # TODO: Discuss the need for check for snowflake password before passing the session + # kwargs to open a snowflake session + # snowflake_password = get_snowflake_password() + # if snowflake_password is not None and snowflake_password: # session_kwargs["password"] = snowflake_password return {k: v for k, v in session_kwargs.items() if v is not None}