update to github actions file to meet new demo structure (#302)

* Update github actions for Development Workflow to match new Connections options. * Added fully_qualified_schema_name in CreateChangeHistoryTable class to allow creation of schema with fully qualified name. * Added script file to use in multiple workflows.
Snowflake-Labs · Nov 22, 2024 · 02e101e · 02e101e
1 parent e878e17
commit 02e101e
Show file tree

Hide file tree

Showing 14 changed files with 110 additions and 102 deletions.
diff --git a/.github/workflows/dev-pytest.yml b/.github/workflows/dev-pytest.yml
@@ -54,35 +54,22 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Create and populate connections.toml
+        run: |
+          echo "Current Directory: ${PWD}"
+          echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}"
+          chmod +x populateConnection.sh
+          bash populateConnection.sh
+        working-directory: .
       - name: Test with pytest
         id: pytest
         run: |
           pytest
       # Testing Schemachange demo projects
       - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema
         run: |
-          echo "::group::Setting up ${MY_TARGET_SCHEMA}"
-          schemachange deploy --config-folder ./demo/setup/${SCENARIO_NAME}
-          echo "::endgroup::"
-          echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}"
-          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/A__render.sql
-          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/R__render.sql
-          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/V1.0.0__render.sql
-          echo "::endgroup::"
-          echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}"
-          set +e
-          schemachange deploy --config-folder ./demo/${SCENARIO_NAME}
-          RESULT=$?
-          if [ $RESULT -eq 0 ]; then
-            echo "Deployment Completed!"
-          else
-            echo "Deployment Failed. Proceeding to Teardown."
-          fi
-          echo "::endgroup::"
-          set -e
-          echo "::group::Tearing down up ${MY_TARGET_SCHEMA}"
-          schemachange deploy --config-folder ./demo/teardown/${SCENARIO_NAME}
-          echo "::endgroup::"
-          if [ $RESULT -ne 0 ]; then
-            exit 1
-          fi
+          echo "Current Directory: ${PWD}"
+          echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}"
+          chmod +x testSchemachange.sh
+          bash testSchemachange.sh
+        working-directory: .
diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml
@@ -60,73 +60,20 @@ jobs:
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Create and populate connections.toml
         run: |
-          touch ./connections.toml
-          echo [default] >> ./connections.toml
-          echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml
-          echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml
-          echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml
-          echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml
-          echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml
-          echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml
-          echo "cat connections.toml"
-          cat ./connections.toml
+          echo "Current Directory: ${PWD}"
+          echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}"
+          chmod +x populateConnection.sh
+          bash populateConnection.sh
+        working-directory: .
       - name: Test with pytest
         id: pytest
         run: |
           pytest
       # Testing Schemachange demo projects
       - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema
         run: |
-          echo "::group::Setting up ${MY_TARGET_SCHEMA}"
-          schemachange deploy \
-            --config-folder ./demo \
-            --config-file-name schemachange-config-setup.yml \
-            --root-folder ./demo/${SCENARIO_NAME}/1_setup \
-            --connection-name default \
-            --connections-file-path ./connections.toml \
-            --verbose
-          echo "::endgroup::"
-
-          echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}"
-
-          schemachange render \
-            --config-folder ./demo/${SCENARIO_NAME} \
-            ./demo/${SCENARIO_NAME}/2_test/A__render.sql
-          schemachange render \
-            --config-folder ./demo/${SCENARIO_NAME} \
-            ./demo/${SCENARIO_NAME}/2_test/R__render.sql
-          schemachange render \
-            --config-folder ./demo/${SCENARIO_NAME} \
-            ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql
-          echo "::endgroup::"
-
-          echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}"
-          set +e
-          schemachange deploy \
-            --config-folder ./demo/${SCENARIO_NAME} \
-            --connection-name default \
-            --connections-file-path ./connections.toml \
-            --root-folder ./demo/${SCENARIO_NAME}/2_test \
-            --verbose
-          RESULT=$?
-          if [ $RESULT -eq 0 ]; then
-            echo "Deployment Completed!"
-          else
-            echo "Deployment Failed. Proceeding to Teardown."
-          fi
-          echo "::endgroup::"
-          set -e
-
-          echo "::group::Tearing down up ${MY_TARGET_SCHEMA}"
-          schemachange deploy \
-            --config-folder ./demo \
-            --config-file-name schemachange-config-teardown.yml \
-            --connection-name default \
-            --connections-file-path ./connections.toml \
-            --root-folder ./demo/${SCENARIO_NAME}/3_teardown \
-            --verbose
-          echo "::endgroup::"
-
-          if [ $RESULT -ne 0 ]; then
-            exit 1
-          fi
+          echo "Current Directory: ${PWD}"
+          echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}"
+          chmod +x testSchemachange.sh
+          bash testSchemachange.sh
+        working-directory: .
diff --git a/demo/basics_demo/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml
@@ -1,6 +1,6 @@
 config-version: 1
 
-root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}"
+root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/2_test"
 
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"
 create-change-history-table: true

diff --git a/demo/citibike_demo/schemachange-config.yml b/demo/citibike_demo/schemachange-config.yml
@@ -1,7 +1,5 @@
 config-version: 1
 
-root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}"
-
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"
 create-change-history-table: true
 

diff --git a/demo/citibike_demo_jinja/schemachange-config.yml b/demo/citibike_demo_jinja/schemachange-config.yml
@@ -1,6 +1,5 @@
 config-version: 1
 
-root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}"
 modules-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/modules"
 
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"

diff --git a/demo/schemachange-config-setup.yml b/demo/schemachange-config-setup.yml
@@ -1,7 +1,5 @@
 config-version: 1
 
-root-folder: "./demo/setup/{{ env_var('SCENARIO_NAME')}}"
-
 # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
 create-change-history-table: true

diff --git a/demo/schemachange-config-teardown.yml b/demo/schemachange-config-teardown.yml
@@ -1,7 +1,5 @@
 config-version: 1
 
-root-folder: "./demo/teardown/{{ env_var('SCENARIO_NAME')}}"
-
 # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
 create-change-history-table: true

diff --git a/populateConnection.sh b/populateConnection.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository.
+touch ./connections.toml
+echo [default] >> ./connections.toml
+echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml
+echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml
+echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml
+echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml
+echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml
+echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml
+echo "cat connections.toml"
+cat ./connections.toml
diff --git a/schemachange/config/ChangeHistoryTable.py b/schemachange/config/ChangeHistoryTable.py
@@ -17,6 +17,10 @@ class ChangeHistoryTable:
     @property
     def fully_qualified(self) -> str:
         return f"{self.database_name}.{self.schema_name}.{self.table_name}"
+
+    @property
+    def fully_qualified_schema_name(self) -> str:
+        return f"{self.database_name}.{self.schema_name}"
 
     @classmethod
     def from_str(cls, table_str: str):

diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py
@@ -6,8 +6,10 @@
 
 from schemachange.config.BaseConfig import BaseConfig
 from schemachange.config.ChangeHistoryTable import ChangeHistoryTable
-from schemachange.config.utils import get_snowflake_identifier_string
-
+from schemachange.config.utils import (
+    get_snowflake_identifier_string,
+    get_snowflake_password,
+)
 
 @dataclasses.dataclass(frozen=True)
 class DeployConfig(BaseConfig):
@@ -82,6 +84,12 @@ def get_session_kwargs(self) -> dict:
             "connection_name": self.connection_name,
             "change_history_table": self.change_history_table,
             "autocommit": self.autocommit,
-            "query_tag": self.query_tag,
+            "query_tag": self.query_tag
         }
+
+        # TODO: Discuss the need for check for snowflake password before passing the session 
+        # kwargs to open a snowflake session
+        # snowflake_password = get_snowflake_password()
+        # if snowflake_password is not None and snowflake_password:
+        #    session_kwargs["password"] = snowflake_password 
         return {k: v for k, v in session_kwargs.items() if v is not None}
diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py
@@ -11,7 +11,7 @@
 from schemachange.config.utils import (
     load_yaml_config,
     validate_directory,
-    validate_file_path,
+    validate_file_path
 )
 
 

diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py
@@ -126,7 +126,7 @@ def parse_cli_args(args) -> dict:
         "-a",
         "--snowflake-account",
         type=str,
-        help="The name of the snowflake account (e.g. xy12345.east-us-2.azure)",
+        help="The name of the snowflake account (e.g. xy12345.east-us-2.azure, xy12345.east-us-2.azure.privatelink, org-accountname, org-accountname.privatelink)",
         required=False,
         action="deprecate",
     )

diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py
@@ -149,7 +149,7 @@ def change_history_schema_exists(self) -> bool:
                 return row[0] > 0
 
     def create_change_history_schema(self, dry_run: bool) -> None:
-        query = f"CREATE SCHEMA IF NOT EXISTS {self.change_history_table.schema_name}"
+        query = f"CREATE SCHEMA IF NOT EXISTS {self.change_history_table.fully_qualified_schema_name}"
         if dry_run:
             self.logger.debug(
                 "Running in dry-run mode. Skipping execution.",

diff --git a/testSchemachange.sh b/testSchemachange.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# Script used in github actions to run test the schemachange functionality against the demo scenarios included in the repository.
+echo "::group::Setting up ${MY_TARGET_SCHEMA}"
+schemachange deploy \
+--config-folder ./demo \
+--config-file-name schemachange-config-setup.yml \
+--root-folder ./demo/${SCENARIO_NAME}/1_setup \
+--connection-name default \
+--connections-file-path ./connections.toml \
+--verbose
+echo "::endgroup::"
+
+echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}"
+
+schemachange render \
+--config-folder ./demo/${SCENARIO_NAME} \
+./demo/${SCENARIO_NAME}/2_test/A__render.sql
+schemachange render \
+--config-folder ./demo/${SCENARIO_NAME} \
+./demo/${SCENARIO_NAME}/2_test/R__render.sql
+schemachange render \
+--config-folder ./demo/${SCENARIO_NAME} \
+./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql
+echo "::endgroup::"
+
+echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}"
+set +e
+schemachange deploy \
+--config-folder ./demo/${SCENARIO_NAME} \
+--connection-name default \
+--connections-file-path ./connections.toml \
+--root-folder ./demo/${SCENARIO_NAME}/2_test \
+--verbose
+RESULT=$?
+
+if [ $RESULT -eq 0 ]; then
+    echo "Deployment Completed!"
+else
+    echo "Deployment Failed. Proceeding to Teardown."
+fi
+    echo "::endgroup::"
+
+set -e
+
+echo "::group::Tearing down up ${MY_TARGET_SCHEMA}"
+schemachange deploy \
+--config-folder ./demo \
+--config-file-name schemachange-config-teardown.yml \
+--connection-name default \
+--connections-file-path ./connections.toml \
+--root-folder ./demo/${SCENARIO_NAME}/3_teardown \
+--verbose
+echo "::endgroup::"
+
+if [ $RESULT -ne 0 ]; then
+    exit 1
+fi