Merge pull request #35 from Snowflake-Labs/snowcli2-update

Snowcli2 updates

.devcontainer/Dockerfile

@@ -4,13 +4,9 @@ USER vscode
 WORKDIR /home/vscode
 
 # Configure SnowSQL
-RUN mkdir .snowsql
-COPY .devcontainer/config .snowsql
-
-# Install SnowSQL
-RUN curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.28-linux_x86_64.bash \
- && SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.28-linux_x86_64.bash \
- && rm snowsql-1.2.28-linux_x86_64.bash
+RUN mkdir .snowflake
+COPY .devcontainer/config.toml .snowflake
+COPY .devcontainer/connections.toml .snowflake
 
 # Create the conda environment
 COPY environment.yml .

.devcontainer/config.toml

@@ -0,0 +1,7 @@
+# Can override the default connection name with an environment variable as follows
+#export SNOWFLAKE_DEFAULT_CONNECTION_NAME="default"
+
+# Only for Snow CLI, can override connection details as follows
+#export SNOWFLAKE_CONNECTIONS_DEFAULT_PASSWORD=""
+
+default_connection_name = "default"

.devcontainer/connections.toml

@@ -0,0 +1,11 @@
+# Can override the default connection name with an environment variable as follows
+#export SNOWFLAKE_DEFAULT_CONNECTION_NAME="default"
+
+[default]
+account = "myaccount"
+user = "myuser"
+password = "mypassword"
+role = "HOL_ROLE"
+warehouse = "HOL_WH"
+database = "HOL_DB"
+schema = "ANALYTICS"

.github/workflows/build_and_deploy.yaml

@@ -26,23 +26,12 @@ jobs:
  - name: Install Python packages
  run: pip install -r requirements.txt
 
- - name: Configure snowcli
- env:
- SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }}
- SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }}
- SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }}
- SNOWSQL_ROLE: ${{ secrets.SNOWSQL_ROLE }}
- SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }}
- SNOWSQL_DATABASE: ${{ secrets.SNOWSQL_DATABASE }}
- run: |
- cd $GITHUB_WORKSPACE
- echo "[connections.dev]" > config
- echo "accountname = $SNOWSQL_ACCOUNT" >> config
- echo "username = $SNOWSQL_USER" >> config
- echo "password = $SNOWSQL_PWD" >> config
- echo "rolename = $SNOWSQL_ROLE" >> config
- echo "warehousename = $SNOWSQL_WAREHOUSE" >> config
- echo "dbname = $SNOWSQL_DATABASE" >> config
-
  - name: Deploy Snowpark apps
+ env:
+ SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+ SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+ SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+ SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+ SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+ SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
  run: python deploy_snowpark_apps.py $GITHUB_WORKSPACE

deploy_snowpark_apps.py

@@ -1,7 +1,9 @@
-import sys;
-import os;
+import sys
+import os
+import yaml
 
-ignore_folders = ['__pycache__', '.ipynb_checkpoints']
+ignore_folders = ['.git', '__pycache__', '.ipynb_checkpoints']
+snowflake_project_config_filename = 'snowflake.yml'
 
 if len(sys.argv) != 2:
  print("Root directory is required")
@@ -16,27 +18,33 @@
  base_name = os.path.basename(directory_path)
 
  # Skip any folders we want to ignore
+ # TODO: Update this logic to skip all subfolders of ignored folder
  if base_name in ignore_folders:
 # print(f"Skipping ignored folder {directory_path}")
  continue
 
- # An app.toml file in the folder is our indication that this folder contains
- # a snowcli Snowpark App
- if not "app.toml" in file_names:
+ # An snowflake.yml file in the folder is our indication that this folder contains
+ # a Snow CLI project
+ if not snowflake_project_config_filename in file_names:
 # print(f"Skipping non-app folder {directory_path}")
  continue
+ print(f"Found Snowflake project in folder {directory_path}")
 
- # Next determine what type of app it is
- app_type = "unknown"
- if "local_connection.py" in file_names:
- app_type = "procedure"
- else:
- app_type = "function"
+ # Read the project config
+ project_settings = {}
+ with open(f"{directory_path}/{snowflake_project_config_filename}", "r") as yamlfile:
+ project_settings = yaml.load(yamlfile, Loader=yaml.FullLoader)
 
- # Finally deploy the app with the snowcli tool
- print(f"Found {app_type} app in folder {directory_path}")
- print(f"Calling snowcli to deploy the {app_type} app")
+ # Confirm that this is a Snowpark project
+ # TODO: Would be better if the project config file had a project_type key!
+ if 'snowpark' not in project_settings:
+ print(f"Skipping non Snowpark project in folder {base_name}")
+ continue
+
+ # Finally deploy the Snowpark project with the snowcli tool
+ print(f"Found Snowflake Snowpark project '{project_settings['snowpark']['project_name']}' in folder {base_name}")
+ print(f"Calling snowcli to deploy the project")
  os.chdir(f"{directory_path}")
- # snow login will update the app.toml file with the correct path to the snowsql config file
- os.system(f"snow login -c {root_directory}/config -C dev")
- os.system(f"snow {app_type} create")
+ # Make sure all 6 SNOWFLAKE_ environment variables are set
+ # SnowCLI accesses the passowrd directly from the SNOWFLAKE_PASSWORD environmnet variable
+ os.system(f"snow snowpark deploy --replace --temporary-connection --account $SNOWFLAKE_ACCOUNT --user $SNOWFLAKE_USER --role $SNOWFLAKE_ROLE --warehouse $SNOWFLAKE_WAREHOUSE --database $SNOWFLAKE_DATABASE")

environment.yml

@@ -9,4 +9,4 @@ dependencies:
  - openssl=1.1.1
  - pip:
  # Snowflake
- - snowflake-cli-labs==0.2.9
+ - snowflake-cli-labs

requirements.txt

@@ -1,2 +1,2 @@
 snowflake-snowpark-python[pandas]
-snowflake-cli-labs==0.2.9
+snowflake-cli-labs

steps/02_load_raw.py

@@ -66,16 +66,7 @@ def validate_raw_tables(session):
 
 # For local debugging
 if __name__ == "__main__":
- # Add the utils package to our path and import the snowpark_utils function
- import os, sys
- current_dir = os.getcwd()
- parent_dir = os.path.dirname(current_dir)
- sys.path.append(parent_dir)
-
- from utils import snowpark_utils
- session = snowpark_utils.get_snowpark_session()
-
- load_all_raw_tables(session)
-# validate_raw_tables(session)
-
- session.close()
+ # Create a local Snowpark session
+ with Session.builder.getOrCreate() as session:
+ load_all_raw_tables(session)
+# validate_raw_tables(session)

steps/04_create_pos_view.py

@@ -106,17 +106,8 @@ def test_pos_view(session):
 
 # For local debugging
 if __name__ == "__main__":
- # Add the utils package to our path and import the snowpark_utils function
- import os, sys
- current_dir = os.getcwd()
- parent_dir = os.path.dirname(current_dir)
- sys.path.append(parent_dir)
-
- from utils import snowpark_utils
- session = snowpark_utils.get_snowpark_session()
-
- create_pos_view(session)
- create_pos_view_stream(session)
-# test_pos_view(session)
-
- session.close()
+ # Create a local Snowpark session
+ with Session.builder.getOrCreate() as session:
+ create_pos_view(session)
+ create_pos_view_stream(session)
+# test_pos_view(session)

steps/05_fahrenheit_to_celsius_udf/.gitignore

@@ -3,3 +3,4 @@
 app.zip
 requirements.snowflake.txt
 __pycache__
+fahrenheit_to_celsius_udf.zip

steps/05_fahrenheit_to_celsius_udf/requirements.txt

@@ -0,0 +1 @@
+snowflake-snowpark-python

steps/05_fahrenheit_to_celsius_udf/snowflake.yml

@@ -0,0 +1,15 @@
+definition_version: 1
+snowpark:
+ project_name: "hol"
+ stage_name: "analytics.deployment"
+ src: "fahrenheit_to_celsius_udf/"
+ functions:
+ - name: "fahrenheit_to_celsius_udf"
+ database: "hol_db"
+ schema: "analytics"
+ handler: "function.main"
+ runtime: "3.10"
+ signature:
+ - name: "temp_f"
+ type: "float"
+ returns: float

steps/06_orders_update_sp/.gitignore

@@ -3,3 +3,4 @@
 app.zip
 requirements.snowflake.txt
 __pycache__
+orders_update_sp.zip

steps/06_orders_update_sp/app.py → steps/06_orders_update_sp/orders_update_sp/procedure.py

@@ -57,18 +57,10 @@ def main(session: Session) -> str:
 # For local debugging
 # Be aware you may need to type-convert arguments if you add input parameters
 if __name__ == '__main__':
- # Add the utils package to our path and import the snowpark_utils function
- import os, sys
- current_dir = os.getcwd()
- parent_parent_dir = os.path.dirname(os.path.dirname(current_dir))
- sys.path.append(parent_parent_dir)
-
- from utils import snowpark_utils
- session = snowpark_utils.get_snowpark_session()
-
- if len(sys.argv) > 1:
- print(main(session, *sys.argv[1:])) # type: ignore
- else:
- print(main(session)) # type: ignore
-
- session.close()
+ # Create a local Snowpark session
+ with Session.builder.getOrCreate() as session:
+ import sys
+ if len(sys.argv) > 1:
+ print(main(session, *sys.argv[1:])) # type: ignore
+ else:
+ print(main(session)) # type: ignore

steps/06_orders_update_sp/requirements.txt

@@ -1,2 +1 @@
 snowflake-snowpark-python
-toml # for local development

steps/06_orders_update_sp/snowflake.yml

@@ -0,0 +1,13 @@
+definition_version: 1
+snowpark:
+ project_name: "hol"
+ stage_name: "harmonized.deployment"
+ src: "orders_update_sp/"
+ procedures:
+ - name: "orders_update_sp"
+ database: "hol_db"
+ schema: "harmonized"
+ handler: "procedure.main"
+ runtime: "3.10"
+ signature: ""
+ returns: string

steps/07_daily_city_metrics_update_sp/.gitignore

@@ -3,3 +3,4 @@
 app.zip
 requirements.snowflake.txt
 __pycache__
+daily_city_metrics_update_sp.zip

steps/07_daily_city_metrics_update_sp/app.py → steps/07_daily_city_metrics_update_sp/daily_city_metrics_update_sp/procedure.py

@@ -105,18 +105,10 @@ def main(session: Session) -> str:
 # For local debugging
 # Be aware you may need to type-convert arguments if you add input parameters
 if __name__ == '__main__':
- # Add the utils package to our path and import the snowpark_utils function
- import os, sys
- current_dir = os.getcwd()
- parent_parent_dir = os.path.dirname(os.path.dirname(current_dir))
- sys.path.append(parent_parent_dir)
-
- from utils import snowpark_utils
- session = snowpark_utils.get_snowpark_session()
-
- if len(sys.argv) > 1:
- print(main(session, *sys.argv[1:])) # type: ignore
- else:
- print(main(session)) # type: ignore
-
- session.close()
+ # Create a local Snowpark session
+ with Session.builder.getOrCreate() as session:
+ import sys
+ if len(sys.argv) > 1:
+ print(main(session, *sys.argv[1:])) # type: ignore
+ else:
+ print(main(session)) # type: ignore