diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml index 94abf1e..e325e82 100644 --- a/.github/workflows/dev-pytest.yml +++ b/.github/workflows/dev-pytest.yml @@ -54,6 +54,13 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Create and populate connections.toml + run: | + echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" + chmod +x populateConnection.sh + bash populateConnection.sh + working-directory: . - name: Test with pytest id: pytest run: | @@ -61,28 +68,8 @@ jobs: # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | - echo "::group::Setting up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/setup/${SCENARIO_NAME} - echo "::endgroup::" - echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/A__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/R__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/V1.0.0__render.sql - echo "::endgroup::" - echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" - set +e - schemachange deploy --config-folder ./demo/${SCENARIO_NAME} - RESULT=$? - if [ $RESULT -eq 0 ]; then - echo "Deployment Completed!" - else - echo "Deployment Failed. Proceeding to Teardown." - fi - echo "::endgroup::" - set -e - echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/teardown/${SCENARIO_NAME} - echo "::endgroup::" - if [ $RESULT -ne 0 ]; then - exit 1 - fi + echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" + chmod +x testSchemachange.sh + bash testSchemachange.sh + working-directory: . \ No newline at end of file diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml index 027a09e..3fe0d00 100644 --- a/.github/workflows/master-pytest.yml +++ b/.github/workflows/master-pytest.yml @@ -60,16 +60,11 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Create and populate connections.toml run: | - touch ./connections.toml - echo [default] >> ./connections.toml - echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml - echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml - echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml - echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml - echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml - echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml - echo "cat connections.toml" - cat ./connections.toml + echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" + chmod +x populateConnection.sh + bash populateConnection.sh + working-directory: . - name: Test with pytest id: pytest run: | @@ -77,56 +72,8 @@ jobs: # Testing Schemachange demo projects - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | - echo "::group::Setting up ${MY_TARGET_SCHEMA}" - schemachange deploy \ - --config-folder ./demo \ - --config-file-name schemachange-config-setup.yml \ - --root-folder ./demo/${SCENARIO_NAME}/1_setup \ - --connection-name default \ - --connections-file-path ./connections.toml \ - --verbose - echo "::endgroup::" - - echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - - schemachange render \ - --config-folder ./demo/${SCENARIO_NAME} \ - ./demo/${SCENARIO_NAME}/2_test/A__render.sql - schemachange render \ - --config-folder ./demo/${SCENARIO_NAME} \ - ./demo/${SCENARIO_NAME}/2_test/R__render.sql - schemachange render \ - --config-folder ./demo/${SCENARIO_NAME} \ - ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql - echo "::endgroup::" - - echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" - set +e - schemachange deploy \ - --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path ./connections.toml \ - --root-folder ./demo/${SCENARIO_NAME}/2_test \ - --verbose - RESULT=$? - if [ $RESULT -eq 0 ]; then - echo "Deployment Completed!" - else - echo "Deployment Failed. Proceeding to Teardown." - fi - echo "::endgroup::" - set -e - - echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" - schemachange deploy \ - --config-folder ./demo \ - --config-file-name schemachange-config-teardown.yml \ - --connection-name default \ - --connections-file-path ./connections.toml \ - --root-folder ./demo/${SCENARIO_NAME}/3_teardown \ - --verbose - echo "::endgroup::" - - if [ $RESULT -ne 0 ]; then - exit 1 - fi + echo "Current Directory: ${PWD}" + echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}" + chmod +x testSchemachange.sh + bash testSchemachange.sh + working-directory: . diff --git a/demo/basics_demo/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml index 7252e33..932b7bc 100644 --- a/demo/basics_demo/schemachange-config.yml +++ b/demo/basics_demo/schemachange-config.yml @@ -1,6 +1,6 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" +root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/citibike_demo/schemachange-config.yml b/demo/citibike_demo/schemachange-config.yml index 7252e33..da58e63 100644 --- a/demo/citibike_demo/schemachange-config.yml +++ b/demo/citibike_demo/schemachange-config.yml @@ -1,7 +1,5 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" - change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/citibike_demo_jinja/schemachange-config.yml b/demo/citibike_demo_jinja/schemachange-config.yml index 88cac3e..e80ca9b 100644 --- a/demo/citibike_demo_jinja/schemachange-config.yml +++ b/demo/citibike_demo_jinja/schemachange-config.yml @@ -1,6 +1,5 @@ config-version: 1 -root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" modules-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/modules" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" diff --git a/demo/schemachange-config-setup.yml b/demo/schemachange-config-setup.yml index e68543c..906978d 100644 --- a/demo/schemachange-config-setup.yml +++ b/demo/schemachange-config-setup.yml @@ -1,7 +1,5 @@ config-version: 1 -root-folder: "./demo/setup/{{ env_var('SCENARIO_NAME')}}" - # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/schemachange-config-teardown.yml b/demo/schemachange-config-teardown.yml index 800415a..906978d 100644 --- a/demo/schemachange-config-teardown.yml +++ b/demo/schemachange-config-teardown.yml @@ -1,7 +1,5 @@ config-version: 1 -root-folder: "./demo/teardown/{{ env_var('SCENARIO_NAME')}}" - # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" create-change-history-table: true diff --git a/populateConnection.sh b/populateConnection.sh new file mode 100644 index 0000000..292039b --- /dev/null +++ b/populateConnection.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository. +touch ./connections.toml +echo [default] >> ./connections.toml +echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml +echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml +echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml +echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml +echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml +echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml +echo "cat connections.toml" +cat ./connections.toml \ No newline at end of file diff --git a/schemachange/config/ChangeHistoryTable.py b/schemachange/config/ChangeHistoryTable.py index ba7645a..7491911 100644 --- a/schemachange/config/ChangeHistoryTable.py +++ b/schemachange/config/ChangeHistoryTable.py @@ -17,6 +17,10 @@ class ChangeHistoryTable: @property def fully_qualified(self) -> str: return f"{self.database_name}.{self.schema_name}.{self.table_name}" + + @property + def fully_qualified_schema_name(self) -> str: + return f"{self.database_name}.{self.schema_name}" @classmethod def from_str(cls, table_str: str): diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index e183470..b42fc52 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -6,8 +6,10 @@ from schemachange.config.BaseConfig import BaseConfig from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.config.utils import get_snowflake_identifier_string - +from schemachange.config.utils import ( + get_snowflake_identifier_string, + get_snowflake_password, +) @dataclasses.dataclass(frozen=True) class DeployConfig(BaseConfig): @@ -82,6 +84,12 @@ def get_session_kwargs(self) -> dict: "connection_name": self.connection_name, "change_history_table": self.change_history_table, "autocommit": self.autocommit, - "query_tag": self.query_tag, + "query_tag": self.query_tag } + + # TODO: Discuss the need for check for snowflake password before passing the session + # kwargs to open a snowflake session + # snowflake_password = get_snowflake_password() + # if snowflake_password is not None and snowflake_password: + # session_kwargs["password"] = snowflake_password return {k: v for k, v in session_kwargs.items() if v is not None} diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index f1a2ad5..604f1f4 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -11,7 +11,7 @@ from schemachange.config.utils import ( load_yaml_config, validate_directory, - validate_file_path, + validate_file_path ) diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index 8b4cd01..4c19840 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -126,7 +126,7 @@ def parse_cli_args(args) -> dict: "-a", "--snowflake-account", type=str, - help="The name of the snowflake account (e.g. xy12345.east-us-2.azure)", + help="The name of the snowflake account (e.g. xy12345.east-us-2.azure, xy12345.east-us-2.azure.privatelink, org-accountname, org-accountname.privatelink)", required=False, action="deprecate", ) diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index ba987f4..1346b76 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -149,7 +149,7 @@ def change_history_schema_exists(self) -> bool: return row[0] > 0 def create_change_history_schema(self, dry_run: bool) -> None: - query = f"CREATE SCHEMA IF NOT EXISTS {self.change_history_table.schema_name}" + query = f"CREATE SCHEMA IF NOT EXISTS {self.change_history_table.fully_qualified_schema_name}" if dry_run: self.logger.debug( "Running in dry-run mode. Skipping execution.", diff --git a/testSchemachange.sh b/testSchemachange.sh new file mode 100644 index 0000000..62e451d --- /dev/null +++ b/testSchemachange.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository. +echo "::group::Setting up ${MY_TARGET_SCHEMA}" +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-setup.yml \ +--root-folder ./demo/${SCENARIO_NAME}/1_setup \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--verbose +echo "::endgroup::" + +echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" + +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/A__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/R__render.sql +schemachange render \ +--config-folder ./demo/${SCENARIO_NAME} \ +./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql +echo "::endgroup::" + +echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" +set +e +schemachange deploy \ +--config-folder ./demo/${SCENARIO_NAME} \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/2_test \ +--verbose +RESULT=$? + +if [ $RESULT -eq 0 ]; then + echo "Deployment Completed!" +else + echo "Deployment Failed. Proceeding to Teardown." +fi + echo "::endgroup::" + +set -e + +echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" +schemachange deploy \ +--config-folder ./demo \ +--config-file-name schemachange-config-teardown.yml \ +--connection-name default \ +--connections-file-path ./connections.toml \ +--root-folder ./demo/${SCENARIO_NAME}/3_teardown \ +--verbose +echo "::endgroup::" + +if [ $RESULT -ne 0 ]; then + exit 1 +fi