From 589b926e953b6791699382d2423b275b3a0d4420 Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Sat, 10 Feb 2024 13:29:02 -0700 Subject: [PATCH 1/8] Initial changes for snowcli 2 migration --- .devcontainer/Dockerfile | 10 +-- .devcontainer/config | 9 -- .devcontainer/config.toml | 7 ++ .devcontainer/connections.toml | 11 +++ deploy_snowpark_apps.py | 42 ++++++---- environment.yml | 2 +- requirements.txt | 2 +- steps/05_fahrenheit_to_celsius_udf/.gitignore | 1 + steps/05_fahrenheit_to_celsius_udf/app.toml | 17 ---- .../fahrenheit_to_celsius_udf}/__init__.py | 0 .../function.py} | 0 .../requirements.txt | 1 + .../snowflake.yml | 13 +++ steps/06_orders_update_sp/.gitignore | 1 + steps/06_orders_update_sp/app.toml | 17 ---- steps/06_orders_update_sp/local_connection.py | 34 -------- .../orders_update_sp/__init__.py | 0 .../{app.py => orders_update_sp/procedure.py} | 15 ++-- steps/06_orders_update_sp/requirements.txt | 1 - steps/06_orders_update_sp/snowflake.yml | 11 +++ .../.gitignore | 1 + .../07_daily_city_metrics_update_sp/app.toml | 17 ---- .../daily_city_metrics_update_sp/__init__.py | 0 .../procedure.py} | 15 ++-- .../local_connection.py | 34 -------- .../requirements.txt | 1 - .../snowflake.yml | 11 +++ utils/snowpark_utils.py | 84 ------------------- 28 files changed, 96 insertions(+), 261 deletions(-) delete mode 100644 .devcontainer/config create mode 100644 .devcontainer/config.toml create mode 100644 .devcontainer/connections.toml delete mode 100644 steps/05_fahrenheit_to_celsius_udf/app.toml rename {utils => steps/05_fahrenheit_to_celsius_udf/fahrenheit_to_celsius_udf}/__init__.py (100%) rename steps/05_fahrenheit_to_celsius_udf/{app.py => fahrenheit_to_celsius_udf/function.py} (100%) create mode 100644 steps/05_fahrenheit_to_celsius_udf/snowflake.yml delete mode 100644 steps/06_orders_update_sp/app.toml delete mode 100644 steps/06_orders_update_sp/local_connection.py create mode 100644 steps/06_orders_update_sp/orders_update_sp/__init__.py rename steps/06_orders_update_sp/{app.py => orders_update_sp/procedure.py} (86%) create mode 100644 steps/06_orders_update_sp/snowflake.yml delete mode 100644 steps/07_daily_city_metrics_update_sp/app.toml create mode 100644 steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/__init__.py rename steps/07_daily_city_metrics_update_sp/{app.py => daily_city_metrics_update_sp/procedure.py} (94%) delete mode 100644 steps/07_daily_city_metrics_update_sp/local_connection.py create mode 100644 steps/07_daily_city_metrics_update_sp/snowflake.yml delete mode 100644 utils/snowpark_utils.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f5daa2c..9af97fb 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -4,13 +4,9 @@ USER vscode WORKDIR /home/vscode # Configure SnowSQL -RUN mkdir .snowsql -COPY .devcontainer/config .snowsql - -# Install SnowSQL -RUN curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.28-linux_x86_64.bash \ - && SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.28-linux_x86_64.bash \ - && rm snowsql-1.2.28-linux_x86_64.bash +RUN mkdir .snowflake +COPY .devcontainer/config.toml .snowflake +COPY .devcontainer/connections.toml .snowflake # Create the conda environment COPY environment.yml . diff --git a/.devcontainer/config b/.devcontainer/config deleted file mode 100644 index 0248c69..0000000 --- a/.devcontainer/config +++ /dev/null @@ -1,9 +0,0 @@ -# SnowSQL config - -[connections.dev] -accountname = myaccount -username = myusername -password = mypassword -rolename = HOL_ROLE -warehousename = HOL_WH -dbname = HOL_DB \ No newline at end of file diff --git a/.devcontainer/config.toml b/.devcontainer/config.toml new file mode 100644 index 0000000..285edcd --- /dev/null +++ b/.devcontainer/config.toml @@ -0,0 +1,7 @@ +# Can override the default connection name with an environment variable as follows +#export SNOWFLAKE_DEFAULT_CONNECTION_NAME="default" + +# Only for Snow CLI, can override connection details as follows +#export SNOWFLAKE_CONNECTIONS_DEFAULT_PASSWORD="" + +default_connection_name = "default" diff --git a/.devcontainer/connections.toml b/.devcontainer/connections.toml new file mode 100644 index 0000000..79871a0 --- /dev/null +++ b/.devcontainer/connections.toml @@ -0,0 +1,11 @@ +# Can override the default connection name with an environment variable as follows +#export SNOWFLAKE_DEFAULT_CONNECTION_NAME="default" + +[default] +account = "myaccount" +user = "myuser" +password = "mypassword" +role = "HOL_ROLE" +warehouse = "HOL_WH" +database = "HOL_DB" +schema = "EXTERNAL" diff --git a/deploy_snowpark_apps.py b/deploy_snowpark_apps.py index 14ca25c..b60a698 100644 --- a/deploy_snowpark_apps.py +++ b/deploy_snowpark_apps.py @@ -1,7 +1,9 @@ -import sys; -import os; +import sys +import os +import yaml -ignore_folders = ['__pycache__', '.ipynb_checkpoints'] +ignore_folders = ['.git', '__pycache__', '.ipynb_checkpoints'] +snowflake_project_config_filename = 'snowflake.yml' if len(sys.argv) != 2: print("Root directory is required") @@ -16,27 +18,31 @@ base_name = os.path.basename(directory_path) # Skip any folders we want to ignore + # TODO: Update this logic to skip all subfolders of ignored folder if base_name in ignore_folders: # print(f"Skipping ignored folder {directory_path}") continue - # An app.toml file in the folder is our indication that this folder contains - # a snowcli Snowpark App - if not "app.toml" in file_names: + # An snowflake.yml file in the folder is our indication that this folder contains + # a Snow CLI project + if not snowflake_project_config_filename in file_names: # print(f"Skipping non-app folder {directory_path}") continue + print(f"Found Snowflake project in folder {directory_path}") - # Next determine what type of app it is - app_type = "unknown" - if "local_connection.py" in file_names: - app_type = "procedure" - else: - app_type = "function" + # Read the project config + project_settings = {} + with open(f"{directory_path}/{snowflake_project_config_filename}", "r") as yamlfile: + project_settings = yaml.load(yamlfile, Loader=yaml.FullLoader) - # Finally deploy the app with the snowcli tool - print(f"Found {app_type} app in folder {directory_path}") - print(f"Calling snowcli to deploy the {app_type} app") + # Confirm that this is a Snowpark project + # TODO: Would be better if the project config file had a project_type key! + if 'snowpark' not in project_settings: + print(f"Skipping non Snowpark project in folder {base_name}") + continue + + # Finally deploy the Snowpark project with the snowcli tool + print(f"Found Snowflake Snowpark project '{project_settings['snowpark']['project_name']}' in folder {base_name}") + print(f"Calling snowcli to deploy the project") os.chdir(f"{directory_path}") - # snow login will update the app.toml file with the correct path to the snowsql config file - os.system(f"snow login -c {root_directory}/config -C dev") - os.system(f"snow {app_type} create") +# os.system(f"snow snowpark deploy") diff --git a/environment.yml b/environment.yml index 78f5492..fb6a477 100644 --- a/environment.yml +++ b/environment.yml @@ -9,4 +9,4 @@ dependencies: - openssl=1.1.1 - pip: # Snowflake - - snowflake-cli-labs==0.2.9 + - snowflake-cli-labs diff --git a/requirements.txt b/requirements.txt index 7364d82..a44644d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ snowflake-snowpark-python[pandas] -snowflake-cli-labs==0.2.9 +snowflake-cli-labs diff --git a/steps/05_fahrenheit_to_celsius_udf/.gitignore b/steps/05_fahrenheit_to_celsius_udf/.gitignore index d7678a6..f6d3d10 100644 --- a/steps/05_fahrenheit_to_celsius_udf/.gitignore +++ b/steps/05_fahrenheit_to_celsius_udf/.gitignore @@ -3,3 +3,4 @@ app.zip requirements.snowflake.txt __pycache__ +fahrenheit_to_celsius_udf.zip diff --git a/steps/05_fahrenheit_to_celsius_udf/app.toml b/steps/05_fahrenheit_to_celsius_udf/app.toml deleted file mode 100644 index bd069e5..0000000 --- a/steps/05_fahrenheit_to_celsius_udf/app.toml +++ /dev/null @@ -1,17 +0,0 @@ -snowsql_config_path = "~/.snowsql/config" -snowsql_connection_name = "dev" - -[default] -input_parameters = "(temp_f float)" -return_type = "float" -file = "app.zip" -name = "fahrenheit_to_celsius_udf" -handler = "app.main" -execute_as_caller = true - -[dev] -database = "HOL_DB" -schema = "ANALYTICS" -warehouse = "HOL_WH" -role = "HOL_ROLE" -overwrite = true diff --git a/utils/__init__.py b/steps/05_fahrenheit_to_celsius_udf/fahrenheit_to_celsius_udf/__init__.py similarity index 100% rename from utils/__init__.py rename to steps/05_fahrenheit_to_celsius_udf/fahrenheit_to_celsius_udf/__init__.py diff --git a/steps/05_fahrenheit_to_celsius_udf/app.py b/steps/05_fahrenheit_to_celsius_udf/fahrenheit_to_celsius_udf/function.py similarity index 100% rename from steps/05_fahrenheit_to_celsius_udf/app.py rename to steps/05_fahrenheit_to_celsius_udf/fahrenheit_to_celsius_udf/function.py diff --git a/steps/05_fahrenheit_to_celsius_udf/requirements.txt b/steps/05_fahrenheit_to_celsius_udf/requirements.txt index e69de29..ed706cf 100644 --- a/steps/05_fahrenheit_to_celsius_udf/requirements.txt +++ b/steps/05_fahrenheit_to_celsius_udf/requirements.txt @@ -0,0 +1 @@ +snowflake-snowpark-python diff --git a/steps/05_fahrenheit_to_celsius_udf/snowflake.yml b/steps/05_fahrenheit_to_celsius_udf/snowflake.yml new file mode 100644 index 0000000..b88cd45 --- /dev/null +++ b/steps/05_fahrenheit_to_celsius_udf/snowflake.yml @@ -0,0 +1,13 @@ +definition_version: 1 +snowpark: + project_name: "hol" + stage_name: "external.dev_deployment" + src: "fahrenheit_to_celsius_udf/" + functions: + - name: "fahrenheit_to_celsius_udf" + handler: "function.main" + runtime: "3.10" + signature: + - name: "temp_f" + type: "float" + returns: float diff --git a/steps/06_orders_update_sp/.gitignore b/steps/06_orders_update_sp/.gitignore index d7678a6..589462d 100644 --- a/steps/06_orders_update_sp/.gitignore +++ b/steps/06_orders_update_sp/.gitignore @@ -3,3 +3,4 @@ app.zip requirements.snowflake.txt __pycache__ +orders_update_sp.zip diff --git a/steps/06_orders_update_sp/app.toml b/steps/06_orders_update_sp/app.toml deleted file mode 100644 index 09d4869..0000000 --- a/steps/06_orders_update_sp/app.toml +++ /dev/null @@ -1,17 +0,0 @@ -snowsql_config_path = "~/.snowsql/config" -snowsql_connection_name = "dev" - -[default] -input_parameters = "()" -return_type = "string" -file = "app.zip" -name = "orders_update_sp" -handler = "app.main" -execute_as_caller = true - -[dev] -database = "HOL_DB" -schema = "HARMONIZED" -warehouse = "HOL_WH" -role = "HOL_ROLE" -overwrite = true diff --git a/steps/06_orders_update_sp/local_connection.py b/steps/06_orders_update_sp/local_connection.py deleted file mode 100644 index 02bb7ee..0000000 --- a/steps/06_orders_update_sp/local_connection.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - -import os -import configparser -from pathlib import Path -import toml - - -def get_dev_config( - environment: str = 'dev', - app_config_path: Path = Path.cwd().joinpath('app.toml'), -) -> dict: - try: - app_config = toml.load(app_config_path) - config = configparser.ConfigParser(inline_comment_prefixes="#") - if app_config['snowsql_config_path'].startswith('~'): - config.read(os.path.expanduser(app_config['snowsql_config_path'])) - else: - config.read(app_config['snowsql_config_path']) - session_config = config[ - 'connections.' + - app_config['snowsql_connection_name'] - ] - session_config_dict = { - k.replace('name', ''): v.strip('"') - for k, v in session_config.items() - } - session_config_dict.update(app_config.get(environment)) # type: ignore - return session_config_dict - except Exception: - raise Exception( - "Error creating snowpark session - be sure you've logged into " - "the SnowCLI and have a valid app.toml file", - ) diff --git a/steps/06_orders_update_sp/orders_update_sp/__init__.py b/steps/06_orders_update_sp/orders_update_sp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/steps/06_orders_update_sp/app.py b/steps/06_orders_update_sp/orders_update_sp/procedure.py similarity index 86% rename from steps/06_orders_update_sp/app.py rename to steps/06_orders_update_sp/orders_update_sp/procedure.py index b130789..a02dfa5 100644 --- a/steps/06_orders_update_sp/app.py +++ b/steps/06_orders_update_sp/orders_update_sp/procedure.py @@ -48,8 +48,8 @@ def main(session: Session) -> str: create_orders_stream(session) # Process data incrementally - merge_order_updates(session) -# session.table('HARMONIZED.ORDERS').limit(5).show() +# merge_order_updates(session) + session.table('HARMONIZED.ORDERS').limit(5).show() return f"Successfully processed ORDERS" @@ -57,15 +57,10 @@ def main(session: Session) -> str: # For local debugging # Be aware you may need to type-convert arguments if you add input parameters if __name__ == '__main__': - # Add the utils package to our path and import the snowpark_utils function - import os, sys - current_dir = os.getcwd() - parent_parent_dir = os.path.dirname(os.path.dirname(current_dir)) - sys.path.append(parent_parent_dir) - - from utils import snowpark_utils - session = snowpark_utils.get_snowpark_session() + # Create a local Snowpark session + session = Session.builder.getOrCreate() + import sys if len(sys.argv) > 1: print(main(session, *sys.argv[1:])) # type: ignore else: diff --git a/steps/06_orders_update_sp/requirements.txt b/steps/06_orders_update_sp/requirements.txt index 75a099e..ed706cf 100644 --- a/steps/06_orders_update_sp/requirements.txt +++ b/steps/06_orders_update_sp/requirements.txt @@ -1,2 +1 @@ snowflake-snowpark-python -toml # for local development diff --git a/steps/06_orders_update_sp/snowflake.yml b/steps/06_orders_update_sp/snowflake.yml new file mode 100644 index 0000000..738b956 --- /dev/null +++ b/steps/06_orders_update_sp/snowflake.yml @@ -0,0 +1,11 @@ +definition_version: 1 +snowpark: + project_name: "hol" + stage_name: "external.dev_deployment" + src: "orders_update_sp/" + procedures: + - name: "orders_update_sp" + handler: "procedure.main" + runtime: "3.10" + signature: "" + returns: string diff --git a/steps/07_daily_city_metrics_update_sp/.gitignore b/steps/07_daily_city_metrics_update_sp/.gitignore index d7678a6..a150f4c 100644 --- a/steps/07_daily_city_metrics_update_sp/.gitignore +++ b/steps/07_daily_city_metrics_update_sp/.gitignore @@ -3,3 +3,4 @@ app.zip requirements.snowflake.txt __pycache__ +daily_city_metrics_update_sp.zip diff --git a/steps/07_daily_city_metrics_update_sp/app.toml b/steps/07_daily_city_metrics_update_sp/app.toml deleted file mode 100644 index 47ba8dd..0000000 --- a/steps/07_daily_city_metrics_update_sp/app.toml +++ /dev/null @@ -1,17 +0,0 @@ -snowsql_config_path = "~/.snowsql/config" -snowsql_connection_name = "dev" - -[default] -input_parameters = "()" -return_type = "string" -file = "app.zip" -name = "daily_city_metrics_update_sp" -handler = "app.main" -execute_as_caller = true - -[dev] -database = "HOL_DB" -schema = "ANALYTICS" -warehouse = "HOL_WH" -role = "HOL_ROLE" -overwrite = true diff --git a/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/__init__.py b/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/steps/07_daily_city_metrics_update_sp/app.py b/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py similarity index 94% rename from steps/07_daily_city_metrics_update_sp/app.py rename to steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py index 5331804..1d68280 100644 --- a/steps/07_daily_city_metrics_update_sp/app.py +++ b/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py @@ -96,8 +96,8 @@ def main(session: Session) -> str: if not table_exists(session, schema='ANALYTICS', name='DAILY_CITY_METRICS'): create_daily_city_metrics_table(session) - merge_daily_city_metrics(session) -# session.table('ANALYTICS.DAILY_CITY_METRICS').limit(5).show() +# merge_daily_city_metrics(session) + session.table('ANALYTICS.DAILY_CITY_METRICS').limit(5).show() return f"Successfully processed DAILY_CITY_METRICS" @@ -105,15 +105,10 @@ def main(session: Session) -> str: # For local debugging # Be aware you may need to type-convert arguments if you add input parameters if __name__ == '__main__': - # Add the utils package to our path and import the snowpark_utils function - import os, sys - current_dir = os.getcwd() - parent_parent_dir = os.path.dirname(os.path.dirname(current_dir)) - sys.path.append(parent_parent_dir) - - from utils import snowpark_utils - session = snowpark_utils.get_snowpark_session() + # Create a local Snowpark session + session = Session.builder.getOrCreate() + import sys if len(sys.argv) > 1: print(main(session, *sys.argv[1:])) # type: ignore else: diff --git a/steps/07_daily_city_metrics_update_sp/local_connection.py b/steps/07_daily_city_metrics_update_sp/local_connection.py deleted file mode 100644 index 02bb7ee..0000000 --- a/steps/07_daily_city_metrics_update_sp/local_connection.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - -import os -import configparser -from pathlib import Path -import toml - - -def get_dev_config( - environment: str = 'dev', - app_config_path: Path = Path.cwd().joinpath('app.toml'), -) -> dict: - try: - app_config = toml.load(app_config_path) - config = configparser.ConfigParser(inline_comment_prefixes="#") - if app_config['snowsql_config_path'].startswith('~'): - config.read(os.path.expanduser(app_config['snowsql_config_path'])) - else: - config.read(app_config['snowsql_config_path']) - session_config = config[ - 'connections.' + - app_config['snowsql_connection_name'] - ] - session_config_dict = { - k.replace('name', ''): v.strip('"') - for k, v in session_config.items() - } - session_config_dict.update(app_config.get(environment)) # type: ignore - return session_config_dict - except Exception: - raise Exception( - "Error creating snowpark session - be sure you've logged into " - "the SnowCLI and have a valid app.toml file", - ) diff --git a/steps/07_daily_city_metrics_update_sp/requirements.txt b/steps/07_daily_city_metrics_update_sp/requirements.txt index 75a099e..ed706cf 100644 --- a/steps/07_daily_city_metrics_update_sp/requirements.txt +++ b/steps/07_daily_city_metrics_update_sp/requirements.txt @@ -1,2 +1 @@ snowflake-snowpark-python -toml # for local development diff --git a/steps/07_daily_city_metrics_update_sp/snowflake.yml b/steps/07_daily_city_metrics_update_sp/snowflake.yml new file mode 100644 index 0000000..b1de4a7 --- /dev/null +++ b/steps/07_daily_city_metrics_update_sp/snowflake.yml @@ -0,0 +1,11 @@ +definition_version: 1 +snowpark: + project_name: "hol" + stage_name: "external.dev_deployment" + src: "daily_city_metrics_update_sp/" + procedures: + - name: "daily_city_metrics_update_sp" + handler: "procedure.main" + runtime: "3.10" + signature: "" + returns: string diff --git a/utils/snowpark_utils.py b/utils/snowpark_utils.py deleted file mode 100644 index c9fd8e8..0000000 --- a/utils/snowpark_utils.py +++ /dev/null @@ -1,84 +0,0 @@ -from snowflake.snowpark import Session -import os -from typing import Optional - -# Class to store a singleton connection option -class SnowflakeConnection(object): - _connection = None - - @property - def connection(self) -> Optional[Session]: - return type(self)._connection - - @connection.setter - def connection(self, val): - type(self)._connection = val - -# Function to return a configured Snowpark session -def get_snowpark_session() -> Session: - # if running in snowflake - if SnowflakeConnection().connection: - # Not sure what this does? - session = SnowflakeConnection().connection - # if running locally with a config file - # TODO: Look for a creds.json style file. This should be the way all snowpark - # related tools work IMO - # if using snowsql config, like snowcli does - elif os.path.exists(os.path.expanduser('~/.snowsql/config')): - snowpark_config = get_snowsql_config() - SnowflakeConnection().connection = Session.builder.configs(snowpark_config).create() - # otherwise configure from environment variables - elif "SNOWSQL_ACCOUNT" in os.environ: - snowpark_config = { - "account": os.environ["SNOWSQL_ACCOUNT"], - "user": os.environ["SNOWSQL_USER"], - "password": os.environ["SNOWSQL_PWD"], - "role": os.environ["SNOWSQL_ROLE"], - "warehouse": os.environ["SNOWSQL_WAREHOUSE"], - "database": os.environ["SNOWSQL_DATABASE"], - "schema": os.environ["SNOWSQL_SCHEMA"] - } - SnowflakeConnection().connection = Session.builder.configs(snowpark_config).create() - - if SnowflakeConnection().connection: - return SnowflakeConnection().connection # type: ignore - else: - raise Exception("Unable to create a Snowpark session") - - -# Mimic the snowcli logic for getting config details, but skip the app.toml processing -# since this will be called outside the snowcli app context. -# TODO: It would be nice to get rid of this entirely and always use creds.json but -# need to update snowcli to make that happen -def get_snowsql_config( - connection_name: str = 'dev', - config_file_path: str = os.path.expanduser('~/.snowsql/config'), -) -> dict: - import configparser - - snowsql_to_snowpark_config_mapping = { - 'account': 'account', - 'accountname': 'account', - 'username': 'user', - 'password': 'password', - 'rolename': 'role', - 'warehousename': 'warehouse', - 'dbname': 'database', - 'schemaname': 'schema' - } - try: - config = configparser.ConfigParser(inline_comment_prefixes="#") - connection_path = 'connections.' + connection_name - - config.read(config_file_path) - session_config = config[connection_path] - # Convert snowsql connection variable names to snowcli ones - session_config_dict = { - snowsql_to_snowpark_config_mapping[k]: v.strip('"') - for k, v in session_config.items() - } - return session_config_dict - except Exception: - raise Exception( - "Error getting snowsql config details" - ) From 8d6340cb24be6ab6317df4480483870c3ba5c2b0 Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Sat, 10 Feb 2024 16:26:21 -0700 Subject: [PATCH 2/8] changes after first run through --- .devcontainer/connections.toml | 2 +- .github/workflows/build_and_deploy.yaml | 14 +++++++------- deploy_snowpark_apps.py | 2 +- steps/02_load_raw.py | 10 ++-------- steps/04_create_pos_view.py | 10 ++-------- .../orders_update_sp/procedure.py | 4 ++-- .../daily_city_metrics_update_sp/procedure.py | 4 ++-- steps/08_orchestrate_jobs.sql | 8 ++++---- steps/09_process_incrementally.sql | 2 +- steps/10_deploy_via_cicd.sql | 2 +- 10 files changed, 23 insertions(+), 35 deletions(-) diff --git a/.devcontainer/connections.toml b/.devcontainer/connections.toml index 79871a0..0acd64c 100644 --- a/.devcontainer/connections.toml +++ b/.devcontainer/connections.toml @@ -8,4 +8,4 @@ password = "mypassword" role = "HOL_ROLE" warehouse = "HOL_WH" database = "HOL_DB" -schema = "EXTERNAL" +schema = "ANALYTICS" diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index f4d5478..bba6a28 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -36,13 +36,13 @@ jobs: SNOWSQL_DATABASE: ${{ secrets.SNOWSQL_DATABASE }} run: | cd $GITHUB_WORKSPACE - echo "[connections.dev]" > config - echo "accountname = $SNOWSQL_ACCOUNT" >> config - echo "username = $SNOWSQL_USER" >> config - echo "password = $SNOWSQL_PWD" >> config - echo "rolename = $SNOWSQL_ROLE" >> config - echo "warehousename = $SNOWSQL_WAREHOUSE" >> config - echo "dbname = $SNOWSQL_DATABASE" >> config + echo "[default]" > config.toml + echo "account = $SNOWSQL_ACCOUNT" >> config.toml + echo "user = $SNOWSQL_USER" >> config.toml + echo "password = $SNOWSQL_PWD" >> config.toml + echo "role = $SNOWSQL_ROLE" >> config.toml + echo "warehouse = $SNOWSQL_WAREHOUSE" >> config.toml + echo "database = $SNOWSQL_DATABASE" >> config.toml - name: Deploy Snowpark apps run: python deploy_snowpark_apps.py $GITHUB_WORKSPACE diff --git a/deploy_snowpark_apps.py b/deploy_snowpark_apps.py index b60a698..d6f8584 100644 --- a/deploy_snowpark_apps.py +++ b/deploy_snowpark_apps.py @@ -45,4 +45,4 @@ print(f"Found Snowflake Snowpark project '{project_settings['snowpark']['project_name']}' in folder {base_name}") print(f"Calling snowcli to deploy the project") os.chdir(f"{directory_path}") -# os.system(f"snow snowpark deploy") + os.system(f"snow --config-file {root_directory}/config.toml snowpark deploy --replace") diff --git a/steps/02_load_raw.py b/steps/02_load_raw.py index f840a90..83fc5bf 100644 --- a/steps/02_load_raw.py +++ b/steps/02_load_raw.py @@ -66,14 +66,8 @@ def validate_raw_tables(session): # For local debugging if __name__ == "__main__": - # Add the utils package to our path and import the snowpark_utils function - import os, sys - current_dir = os.getcwd() - parent_dir = os.path.dirname(current_dir) - sys.path.append(parent_dir) - - from utils import snowpark_utils - session = snowpark_utils.get_snowpark_session() + # Create a local Snowpark session + session = Session.builder.getOrCreate() load_all_raw_tables(session) # validate_raw_tables(session) diff --git a/steps/04_create_pos_view.py b/steps/04_create_pos_view.py index f66ee30..d5a2a0a 100644 --- a/steps/04_create_pos_view.py +++ b/steps/04_create_pos_view.py @@ -106,14 +106,8 @@ def test_pos_view(session): # For local debugging if __name__ == "__main__": - # Add the utils package to our path and import the snowpark_utils function - import os, sys - current_dir = os.getcwd() - parent_dir = os.path.dirname(current_dir) - sys.path.append(parent_dir) - - from utils import snowpark_utils - session = snowpark_utils.get_snowpark_session() + # Create a local Snowpark session + session = Session.builder.getOrCreate() create_pos_view(session) create_pos_view_stream(session) diff --git a/steps/06_orders_update_sp/orders_update_sp/procedure.py b/steps/06_orders_update_sp/orders_update_sp/procedure.py index a02dfa5..1f594ac 100644 --- a/steps/06_orders_update_sp/orders_update_sp/procedure.py +++ b/steps/06_orders_update_sp/orders_update_sp/procedure.py @@ -48,8 +48,8 @@ def main(session: Session) -> str: create_orders_stream(session) # Process data incrementally -# merge_order_updates(session) - session.table('HARMONIZED.ORDERS').limit(5).show() + merge_order_updates(session) +# session.table('HARMONIZED.ORDERS').limit(5).show() return f"Successfully processed ORDERS" diff --git a/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py b/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py index 1d68280..4a4f5b2 100644 --- a/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py +++ b/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py @@ -96,8 +96,8 @@ def main(session: Session) -> str: if not table_exists(session, schema='ANALYTICS', name='DAILY_CITY_METRICS'): create_daily_city_metrics_table(session) -# merge_daily_city_metrics(session) - session.table('ANALYTICS.DAILY_CITY_METRICS').limit(5).show() + merge_daily_city_metrics(session) +# session.table('ANALYTICS.DAILY_CITY_METRICS').limit(5).show() return f"Successfully processed DAILY_CITY_METRICS" diff --git a/steps/08_orchestrate_jobs.sql b/steps/08_orchestrate_jobs.sql index fec46fe..201c0cd 100644 --- a/steps/08_orchestrate_jobs.sql +++ b/steps/08_orchestrate_jobs.sql @@ -10,7 +10,7 @@ Last Updated: 1/9/2023 USE ROLE HOL_ROLE; USE WAREHOUSE HOL_WH; -USE SCHEMA HOL_DB.HARMONIZED; +USE SCHEMA HOL_DB.ANALYTICS; -- ---------------------------------------------------------------------------- @@ -20,15 +20,15 @@ USE SCHEMA HOL_DB.HARMONIZED; CREATE OR REPLACE TASK ORDERS_UPDATE_TASK WAREHOUSE = HOL_WH WHEN - SYSTEM$STREAM_HAS_DATA('POS_FLATTENED_V_STREAM') + SYSTEM$STREAM_HAS_DATA('HARMONIZED.POS_FLATTENED_V_STREAM') AS -CALL HARMONIZED.ORDERS_UPDATE_SP(); +CALL ANALYTICS.ORDERS_UPDATE_SP(); CREATE OR REPLACE TASK DAILY_CITY_METRICS_UPDATE_TASK WAREHOUSE = HOL_WH AFTER ORDERS_UPDATE_TASK WHEN - SYSTEM$STREAM_HAS_DATA('ORDERS_STREAM') + SYSTEM$STREAM_HAS_DATA('HARMONIZED.ORDERS_STREAM') AS CALL ANALYTICS.DAILY_CITY_METRICS_UPDATE_SP(); diff --git a/steps/09_process_incrementally.sql b/steps/09_process_incrementally.sql index 59b630e..281d126 100644 --- a/steps/09_process_incrementally.sql +++ b/steps/09_process_incrementally.sql @@ -38,7 +38,7 @@ ALTER WAREHOUSE HOL_WH SET WAREHOUSE_SIZE = XSMALL; -- Step #2: Execute the tasks -- ---------------------------------------------------------------------------- -USE SCHEMA HARMONIZED; +USE SCHEMA ANALYTICS; EXECUTE TASK ORDERS_UPDATE_TASK; diff --git a/steps/10_deploy_via_cicd.sql b/steps/10_deploy_via_cicd.sql index 6fe139c..4be78d4 100644 --- a/steps/10_deploy_via_cicd.sql +++ b/steps/10_deploy_via_cicd.sql @@ -28,7 +28,7 @@ Last Updated: 1/9/2023 -- In the terminal change to the 05_fahrenheit_to_celsius_udf directory and run these commands -- pip install -r requirements.txt --- python app.py 35 +-- python fahrenheit_to_celsius_udf/function.py 35 -- ---------------------------------------------------------------------------- From d1c388e3e43243dd76c803f0008b9d271e9e78a3 Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Sat, 10 Feb 2024 17:12:06 -0700 Subject: [PATCH 3/8] Updates to the github actions pipeline for snowcli2 --- .github/workflows/build_and_deploy.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index bba6a28..11e328e 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -28,21 +28,21 @@ jobs: - name: Configure snowcli env: - SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} - SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} - SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} - SNOWSQL_ROLE: ${{ secrets.SNOWSQL_ROLE }} - SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} - SNOWSQL_DATABASE: ${{ secrets.SNOWSQL_DATABASE }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} run: | cd $GITHUB_WORKSPACE echo "[default]" > config.toml - echo "account = $SNOWSQL_ACCOUNT" >> config.toml - echo "user = $SNOWSQL_USER" >> config.toml - echo "password = $SNOWSQL_PWD" >> config.toml - echo "role = $SNOWSQL_ROLE" >> config.toml - echo "warehouse = $SNOWSQL_WAREHOUSE" >> config.toml - echo "database = $SNOWSQL_DATABASE" >> config.toml + echo "account = $SNOWFLAKE_ACCOUNT" >> config.toml + echo "user = $SNOWFLAKE_USER" >> config.toml + echo "password = $SNOWFLAKE_PASSWORD" >> config.toml + echo "role = $SNOWFLAKE_ROLE" >> config.toml + echo "warehouse = $SNOWFLAKE_WAREHOUSE" >> config.toml + echo "database = $SNOWFLAKE_DATABASE" >> config.toml - name: Deploy Snowpark apps run: python deploy_snowpark_apps.py $GITHUB_WORKSPACE From 4a979423898d4353f807f9e45d0da4c5a6ef681d Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Thu, 7 Mar 2024 12:25:26 -0700 Subject: [PATCH 4/8] SnowCLI 2.1 changes --- .github/workflows/build_and_deploy.yaml | 13 +------------ deploy_snowpark_apps.py | 4 +++- steps/05_fahrenheit_to_celsius_udf/snowflake.yml | 2 ++ .../orders_update_sp/procedure.py | 15 ++++++--------- steps/06_orders_update_sp/snowflake.yml | 2 ++ .../daily_city_metrics_update_sp/procedure.py | 15 ++++++--------- .../07_daily_city_metrics_update_sp/snowflake.yml | 2 ++ steps/08_orchestrate_jobs.sql | 8 ++++---- steps/09_process_incrementally.sql | 2 +- 9 files changed, 27 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index 11e328e..cd1441c 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -26,7 +26,7 @@ jobs: - name: Install Python packages run: pip install -r requirements.txt - - name: Configure snowcli + - name: Deploy Snowpark apps env: SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} @@ -34,15 +34,4 @@ jobs: SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} - run: | - cd $GITHUB_WORKSPACE - echo "[default]" > config.toml - echo "account = $SNOWFLAKE_ACCOUNT" >> config.toml - echo "user = $SNOWFLAKE_USER" >> config.toml - echo "password = $SNOWFLAKE_PASSWORD" >> config.toml - echo "role = $SNOWFLAKE_ROLE" >> config.toml - echo "warehouse = $SNOWFLAKE_WAREHOUSE" >> config.toml - echo "database = $SNOWFLAKE_DATABASE" >> config.toml - - - name: Deploy Snowpark apps run: python deploy_snowpark_apps.py $GITHUB_WORKSPACE diff --git a/deploy_snowpark_apps.py b/deploy_snowpark_apps.py index d6f8584..98d82a5 100644 --- a/deploy_snowpark_apps.py +++ b/deploy_snowpark_apps.py @@ -45,4 +45,6 @@ print(f"Found Snowflake Snowpark project '{project_settings['snowpark']['project_name']}' in folder {base_name}") print(f"Calling snowcli to deploy the project") os.chdir(f"{directory_path}") - os.system(f"snow --config-file {root_directory}/config.toml snowpark deploy --replace") + # Make sure all 6 SNOWFLAKE_ environment variables are set + # SnowCLI accesses the passowrd directly from the SNOWFLAKE_PASSWORD environmnet variable + os.system(f"snow snowpark deploy --replace --temporary-connection --account $SNOWFLAKE_ACCOUNT --user $SNOWFLAKE_USER --role $SNOWFLAKE_ROLE --warehouse $SNOWFLAKE_WAREHOUSE --database $SNOWFLAKE_DATABASE") diff --git a/steps/05_fahrenheit_to_celsius_udf/snowflake.yml b/steps/05_fahrenheit_to_celsius_udf/snowflake.yml index b88cd45..46afc86 100644 --- a/steps/05_fahrenheit_to_celsius_udf/snowflake.yml +++ b/steps/05_fahrenheit_to_celsius_udf/snowflake.yml @@ -5,6 +5,8 @@ snowpark: src: "fahrenheit_to_celsius_udf/" functions: - name: "fahrenheit_to_celsius_udf" + database: "hol_db" + schema: "analytics" handler: "function.main" runtime: "3.10" signature: diff --git a/steps/06_orders_update_sp/orders_update_sp/procedure.py b/steps/06_orders_update_sp/orders_update_sp/procedure.py index 1f594ac..4c07640 100644 --- a/steps/06_orders_update_sp/orders_update_sp/procedure.py +++ b/steps/06_orders_update_sp/orders_update_sp/procedure.py @@ -58,12 +58,9 @@ def main(session: Session) -> str: # Be aware you may need to type-convert arguments if you add input parameters if __name__ == '__main__': # Create a local Snowpark session - session = Session.builder.getOrCreate() - - import sys - if len(sys.argv) > 1: - print(main(session, *sys.argv[1:])) # type: ignore - else: - print(main(session)) # type: ignore - - session.close() + with Session.builder.getOrCreate() as session: + import sys + if len(sys.argv) > 1: + print(main(session, *sys.argv[1:])) # type: ignore + else: + print(main(session)) # type: ignore diff --git a/steps/06_orders_update_sp/snowflake.yml b/steps/06_orders_update_sp/snowflake.yml index 738b956..d3c9730 100644 --- a/steps/06_orders_update_sp/snowflake.yml +++ b/steps/06_orders_update_sp/snowflake.yml @@ -5,6 +5,8 @@ snowpark: src: "orders_update_sp/" procedures: - name: "orders_update_sp" + database: "hol_db" + schema: "harmonized" handler: "procedure.main" runtime: "3.10" signature: "" diff --git a/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py b/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py index 4a4f5b2..ca12291 100644 --- a/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py +++ b/steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py @@ -106,12 +106,9 @@ def main(session: Session) -> str: # Be aware you may need to type-convert arguments if you add input parameters if __name__ == '__main__': # Create a local Snowpark session - session = Session.builder.getOrCreate() - - import sys - if len(sys.argv) > 1: - print(main(session, *sys.argv[1:])) # type: ignore - else: - print(main(session)) # type: ignore - - session.close() + with Session.builder.getOrCreate() as session: + import sys + if len(sys.argv) > 1: + print(main(session, *sys.argv[1:])) # type: ignore + else: + print(main(session)) # type: ignore diff --git a/steps/07_daily_city_metrics_update_sp/snowflake.yml b/steps/07_daily_city_metrics_update_sp/snowflake.yml index b1de4a7..4accf82 100644 --- a/steps/07_daily_city_metrics_update_sp/snowflake.yml +++ b/steps/07_daily_city_metrics_update_sp/snowflake.yml @@ -5,6 +5,8 @@ snowpark: src: "daily_city_metrics_update_sp/" procedures: - name: "daily_city_metrics_update_sp" + database: "hol_db" + schema: "analytics" handler: "procedure.main" runtime: "3.10" signature: "" diff --git a/steps/08_orchestrate_jobs.sql b/steps/08_orchestrate_jobs.sql index 201c0cd..fec46fe 100644 --- a/steps/08_orchestrate_jobs.sql +++ b/steps/08_orchestrate_jobs.sql @@ -10,7 +10,7 @@ Last Updated: 1/9/2023 USE ROLE HOL_ROLE; USE WAREHOUSE HOL_WH; -USE SCHEMA HOL_DB.ANALYTICS; +USE SCHEMA HOL_DB.HARMONIZED; -- ---------------------------------------------------------------------------- @@ -20,15 +20,15 @@ USE SCHEMA HOL_DB.ANALYTICS; CREATE OR REPLACE TASK ORDERS_UPDATE_TASK WAREHOUSE = HOL_WH WHEN - SYSTEM$STREAM_HAS_DATA('HARMONIZED.POS_FLATTENED_V_STREAM') + SYSTEM$STREAM_HAS_DATA('POS_FLATTENED_V_STREAM') AS -CALL ANALYTICS.ORDERS_UPDATE_SP(); +CALL HARMONIZED.ORDERS_UPDATE_SP(); CREATE OR REPLACE TASK DAILY_CITY_METRICS_UPDATE_TASK WAREHOUSE = HOL_WH AFTER ORDERS_UPDATE_TASK WHEN - SYSTEM$STREAM_HAS_DATA('HARMONIZED.ORDERS_STREAM') + SYSTEM$STREAM_HAS_DATA('ORDERS_STREAM') AS CALL ANALYTICS.DAILY_CITY_METRICS_UPDATE_SP(); diff --git a/steps/09_process_incrementally.sql b/steps/09_process_incrementally.sql index 281d126..59b630e 100644 --- a/steps/09_process_incrementally.sql +++ b/steps/09_process_incrementally.sql @@ -38,7 +38,7 @@ ALTER WAREHOUSE HOL_WH SET WAREHOUSE_SIZE = XSMALL; -- Step #2: Execute the tasks -- ---------------------------------------------------------------------------- -USE SCHEMA ANALYTICS; +USE SCHEMA HARMONIZED; EXECUTE TASK ORDERS_UPDATE_TASK; From 0759102aafe2140accc7fcff458d1f2ec7498505 Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Thu, 7 Mar 2024 12:31:02 -0700 Subject: [PATCH 5/8] Simplified debugging blocks --- steps/02_load_raw.py | 9 +++------ steps/04_create_pos_view.py | 11 ++++------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/steps/02_load_raw.py b/steps/02_load_raw.py index 83fc5bf..b6eff31 100644 --- a/steps/02_load_raw.py +++ b/steps/02_load_raw.py @@ -67,9 +67,6 @@ def validate_raw_tables(session): # For local debugging if __name__ == "__main__": # Create a local Snowpark session - session = Session.builder.getOrCreate() - - load_all_raw_tables(session) -# validate_raw_tables(session) - - session.close() + with Session.builder.getOrCreate() as session: + load_all_raw_tables(session) +# validate_raw_tables(session) diff --git a/steps/04_create_pos_view.py b/steps/04_create_pos_view.py index d5a2a0a..df8aac3 100644 --- a/steps/04_create_pos_view.py +++ b/steps/04_create_pos_view.py @@ -107,10 +107,7 @@ def test_pos_view(session): # For local debugging if __name__ == "__main__": # Create a local Snowpark session - session = Session.builder.getOrCreate() - - create_pos_view(session) - create_pos_view_stream(session) -# test_pos_view(session) - - session.close() + with Session.builder.getOrCreate() as session: + create_pos_view(session) + create_pos_view_stream(session) +# test_pos_view(session) From 3c348b08d6960073a329228594bdead89aa8cf5b Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Thu, 7 Mar 2024 12:34:11 -0700 Subject: [PATCH 6/8] temporarily pin cli version --- environment.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index fb6a477..d37de89 100644 --- a/environment.yml +++ b/environment.yml @@ -9,4 +9,4 @@ dependencies: - openssl=1.1.1 - pip: # Snowflake - - snowflake-cli-labs + - snowflake-cli-labs==2.1.0rc1 diff --git a/requirements.txt b/requirements.txt index a44644d..c8e749a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ snowflake-snowpark-python[pandas] -snowflake-cli-labs +snowflake-cli-labs==2.1.0rc1 From 29462ea54e7f6b0d052475c8e31f6cef2af60a20 Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Thu, 7 Mar 2024 14:15:23 -0700 Subject: [PATCH 7/8] Updates to snowpark stage --- steps/05_fahrenheit_to_celsius_udf/snowflake.yml | 2 +- steps/06_orders_update_sp/snowflake.yml | 2 +- steps/07_daily_city_metrics_update_sp/snowflake.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/steps/05_fahrenheit_to_celsius_udf/snowflake.yml b/steps/05_fahrenheit_to_celsius_udf/snowflake.yml index 46afc86..616a139 100644 --- a/steps/05_fahrenheit_to_celsius_udf/snowflake.yml +++ b/steps/05_fahrenheit_to_celsius_udf/snowflake.yml @@ -1,7 +1,7 @@ definition_version: 1 snowpark: project_name: "hol" - stage_name: "external.dev_deployment" + stage_name: "analytics.deployment" src: "fahrenheit_to_celsius_udf/" functions: - name: "fahrenheit_to_celsius_udf" diff --git a/steps/06_orders_update_sp/snowflake.yml b/steps/06_orders_update_sp/snowflake.yml index d3c9730..402f147 100644 --- a/steps/06_orders_update_sp/snowflake.yml +++ b/steps/06_orders_update_sp/snowflake.yml @@ -1,7 +1,7 @@ definition_version: 1 snowpark: project_name: "hol" - stage_name: "external.dev_deployment" + stage_name: "harmonized.deployment" src: "orders_update_sp/" procedures: - name: "orders_update_sp" diff --git a/steps/07_daily_city_metrics_update_sp/snowflake.yml b/steps/07_daily_city_metrics_update_sp/snowflake.yml index 4accf82..4874dcb 100644 --- a/steps/07_daily_city_metrics_update_sp/snowflake.yml +++ b/steps/07_daily_city_metrics_update_sp/snowflake.yml @@ -1,7 +1,7 @@ definition_version: 1 snowpark: project_name: "hol" - stage_name: "external.dev_deployment" + stage_name: "analytics.deployment" src: "daily_city_metrics_update_sp/" procedures: - name: "daily_city_metrics_update_sp" From 64025cf4181ba37b6be8a78775085e309b419b7a Mon Sep 17 00:00:00 2001 From: Jeremiah Hansen Date: Thu, 7 Mar 2024 14:26:02 -0700 Subject: [PATCH 8/8] removed pinned version --- environment.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index d37de89..fb6a477 100644 --- a/environment.yml +++ b/environment.yml @@ -9,4 +9,4 @@ dependencies: - openssl=1.1.1 - pip: # Snowflake - - snowflake-cli-labs==2.1.0rc1 + - snowflake-cli-labs diff --git a/requirements.txt b/requirements.txt index c8e749a..a44644d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ snowflake-snowpark-python[pandas] -snowflake-cli-labs==2.1.0rc1 +snowflake-cli-labs