Fix for glob patterns in artifacts

RELEASE-NOTES.md

@@ -44,6 +44,7 @@
 * Added templates expansion of arbitrary files for Native Apps through `templates` processor.
 * Added `SNOWFLAKE_..._PRIVATE_KEY_RAW` environment variable to pass private key as a raw string.
 * Added periodic check for newest version of Snowflake CLI. When new version is available, user will be notified.
+* Snowpark and Streamlit entities in `snowflake.yml` now accept glob patterns for listing artifacts
 
 ## Fixes and improvements
 * Fixed problem with whitespaces in `snow connection add` command.

src/snowflake/cli/_plugins/snowpark/commands.py

@@ -189,7 +189,7 @@ def validate_all_artifacts_exists(
  project_paths: SnowparkProjectPaths, snowpark_entities: SnowparkEntities
 ):
  for key, entity in snowpark_entities.items():
- for artefact in entity.artifacts:
+ for artefact in entity.get_artifacts():
  path = project_paths.get_artefact_dto(artefact).post_build_path
  if not path.exists():
  raise UsageError(
@@ -219,7 +219,7 @@ def build_artifacts_mappings(
  for entity_id, entity in snowpark_entities.items():
  stage = entity.stage
  required_artifacts = set()
- for artefact in entity.artifacts:
+ for artefact in entity.get_artifacts():
  artefact_dto = project_paths.get_artefact_dto(artefact)
  required_artifacts.add(artefact_dto)
  entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage))

src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py

@@ -82,13 +82,27 @@ def convert_runtime(cls, runtime_input: Union[str, float]) -> str:
  return str(runtime_input)
  return runtime_input
 
- @field_validator("artifacts")
- @classmethod
- def validate_artifacts(cls, artifacts: List[Path]) -> List[Path]:
- for artefact in artifacts:
- if "*" in str(artefact):
- raise ValueError("Glob patterns not supported for Snowpark artifacts.")
- return artifacts
+ def get_artifacts(self) -> List[PathMapping]:
+ _artifacts = []
+ for artifact in self.artifacts:
+ if isinstance(artifact, str):
+ _artifacts.append(PathMapping(src=Path(artifact)))
+ elif isinstance(artifact, PathMapping):
+ if "*" in str(artifact.src):
+ root = (
+ artifact.src.parent.absolute()
+ if not "**" in str(artifact)
+ else Path(".").absolute()
+ )
+ _artifacts.extend(
+ [
+ PathMapping(src=item, dest=artifact.dest)
+ for item in root.glob(artifact.src.name)
+ ]
+ )
+ else:
+ _artifacts.append(artifact)
+ return _artifacts
 
  @property
  def udf_sproc_identifier(self) -> UdfSprocIdentifier:

src/snowflake/cli/_plugins/streamlit/manager.py

@@ -173,7 +173,7 @@ def deploy(self, streamlit: StreamlitEntityModel, replace: bool = False):
 
  self._put_streamlit_files(
  root_location,
- streamlit.artifacts,
+ streamlit.get_artifacts(),
  )
  else:
  """
@@ -193,7 +193,7 @@ def deploy(self, streamlit: StreamlitEntityModel, replace: bool = False):
  f"{stage_name}/{streamlit_name_for_root_location}"
  )
 
- self._put_streamlit_files(root_location, streamlit.artifacts)
+ self._put_streamlit_files(root_location, streamlit.get_artifacts())
 
  self._create_streamlit(
  streamlit=streamlit,

src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py

@@ -66,9 +66,24 @@ def artifacts_must_exists(self):
  return self
 
  for artifact in self.artifacts:
- if not artifact.exists():
+
+ if "*" not in artifact.name and not artifact.exists():
  raise ValueError(
  f"Specified artifact {artifact} does not exist locally."
  )
-
  return self
+
+ def get_artifacts(self):
+ _artifacts = []
+ for artifact in self.artifacts:
+ if "*" in str(artifact):
+ root = (
+ artifact.parent.absolute()
+ if "**" not in str(artifact)
+ else Path(".")
+ )
+ _artifacts.extend([item for item in root.glob(artifact.name)])
+ else:
+ _artifacts.append(artifact)
+
+ return _artifacts

tests/snowpark/test_build.py

@@ -28,3 +28,9 @@ def test_snowpark_build_no_deprecated_warnings_by_default(
  result = runner.invoke(["snowpark", "build", "--ignore-anaconda"])
  assert result.exit_code == 0, result.output
  assert "flag is deprecated" not in result.output
+
+
+def test_build_with_glob_patterns_in_artifacts(runner, project_directory):
+ with project_directory("glob_patterns"):
+ result = runner.invoke(["snowpark", "build", "--ignore-anaconda"])
+ assert result.exit_code == 0

tests/snowpark/test_procedure.py

@@ -547,3 +547,42 @@ def test_snowpark_fail_if_no_active_warehouse(runner, mock_ctx, project_director
  "The command requires warehouse. No warehouse found in current connection."
  in result.output
  )
+
+
+@mock.patch("snowflake.connector.connect")
+@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe")
+@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show")
+@mock.patch("snowflake.cli._plugins.snowpark.commands.StageManager.put")
+@mock_session_has_warehouse
+def test_deploy_procedure_with_glob_patterns_in_src(
+ mock_sm_put,
+ mock_om_show,
+ mock_om_describe,
+ mock_conn,
+ runner,
+ mock_ctx,
+ project_directory,
+):
+ mock_om_describe.side_effect = ProgrammingError(
+ errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED
+ )
+ ctx = mock_ctx()
+ mock_conn.return_value = ctx
+
+ with project_directory("glob_patterns") as tmp:
+
+ result = runner.invoke(
+ [
+ "snowpark",
+ "deploy",
+ ]
+ )
+ assert result.exit_code == 0, result.output
+ assert any(
+ "app.py" in str(call.kwargs.get("local_path"))
+ for call in mock_sm_put.mock_calls
+ )
+ assert any(
+ "procedures.py" in str(call.kwargs.get("local_path"))
+ for call in mock_sm_put.mock_calls
+ )

tests/streamlit/test_commands.py

@@ -954,3 +954,27 @@ def test_deploy_streamlit_with_comment_v2(
  REGIONLESS_QUERY,
  "select current_account_name()",
  ]
+
+
+@mock.patch("snowflake.connector.connect")
+@mock.patch("snowflake.cli._plugins.streamlit.manager.StageManager")
+def test_deploy_streamlit_with_glob_pattern_in_artifacts(
+ mock_sm_put, mock_connector, runner, project_directory, mock_ctx, mock_cursor
+):
+ ctx = mock_ctx(
+ mock_cursor(
+ rows=[
+ {"SYSTEM$GET_SNOWSIGHT_HOST()": "https://snowsight.domain"},
+ {"REGIONLESS": "false"},
+ {"CURRENT_ACCOUNT_NAME()": "https://snowsight.domain"},
+ ],
+ columns=["SYSTEM$GET_SNOWSIGHT_HOST()"],
+ )
+ )
+ mock_connector.return_value = ctx
+ with project_directory("glob_patterns"):
+ result = runner.invoke(["streamlit", "deploy", "--replace"])
+
+ assert result.exit_code == 0, result.output
+ assert any("my_page.py" in str(call[1]) for call in mock_sm_put.mock_calls)
+ assert any("my_page2.py" in str(call[1]) for call in mock_sm_put.mock_calls)

tests/test_data/projects/glob_patterns/environment.yml

@@ -0,0 +1,5 @@
+name: sf_env
+channels:
+ - snowflake
+dependencies:
+ - pandas

tests/test_data/projects/glob_patterns/pages/my_page.py

@@ -0,0 +1,3 @@
+import streamlit as st
+
+st.title("Example page")

tests/test_data/projects/glob_patterns/pages/my_page2.py

@@ -0,0 +1,3 @@
+import streamlit as st
+
+st.title("Another page")

tests/test_data/projects/glob_patterns/snowflake.yml

@@ -0,0 +1,25 @@
+definition_version: 2
+entities:
+ hello_procedure:
+ artifacts:
+ - src/*.*
+ handler: hello
+ identifier:
+ name: hello_function
+ returns: string
+ signature:
+ - name: "name"
+ type: "string"
+ stage: dev_deployment
+ type: procedure
+ my_streamlit:
+ type: "streamlit"
+ identifier: test_streamlit_deploy_snowcli
+ title: "My Fancy Streamlit"
+ stage: streamlit
+ query_warehouse: xsmall
+ main_file: streamlit_app.py
+ artifacts:
+ - streamlit_app.py
+ - pages/*.*
+ - environment.yml

tests/test_data/projects/glob_patterns/src/app.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import sys
+
+from procedures import hello_procedure
+from snowflake.snowpark import Session
+
+# For local debugging. Be aware you may need to type-convert arguments if
+# you add input parameters
+if __name__ == "__main__":
+ from snowflake.cli.api.config import cli_config
+
+ session = Session.builder.configs(cli_config.get_connection_dict("dev")).create()
+ if len(sys.argv) > 1:
+ print(hello_procedure(session, *sys.argv[1:])) # type: ignore
+ else:
+ print(hello_procedure(session)) # type: ignore
+ session.close()

tests/test_data/projects/glob_patterns/src/procedures.py

@@ -0,0 +1,9 @@
+from snowflake.snowpark import Session
+
+
+def hello_procedure(session: Session, name: str) -> str:
+ return f"Hello {name}"
+
+
+def test_procedure(session: Session) -> str:
+ return "Test procedure"

tests/test_data/projects/glob_patterns/streamlit_app.py

@@ -0,0 +1,3 @@
+import streamlit as st
+
+st.title("Example streamlit app")

tests_integration/test_data/projects/snowpark_glob_patterns/app_1/a.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+from snowflake.snowpark import Session
+from b import test_procedure
+
+
+# test import
+import syrupy
+
+
+def hello_procedure(session: Session, name: str) -> str:
+
+ return f"Hello {name}" + test_procedure(session)

tests_integration/test_data/projects/snowpark_glob_patterns/app_1/b.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+from snowflake.snowpark import Session
+
+
+# test import
+import syrupy
+
+
+def test_procedure(session: Session) -> str:
+ return "Test procedure"

tests_integration/test_data/projects/snowpark_glob_patterns/requirements.txt

@@ -0,0 +1 @@
+snowflake-snowpark-pythonsyrupy

tests_integration/test_data/projects/snowpark_glob_patterns/snowflake.yml

@@ -0,0 +1,43 @@
+definition_version: 2
+
+mixins:
+ snowpark_shared:
+ stage: "dev_deployment"
+
+entities:
+ hello_procedure:
+ type: "procedure"
+ stage: "stage_a"
+ identifier:
+ name: "hello_procedure"
+ handler: "a.hello_procedure"
+ signature:
+ - name: "name"
+ type: "string"
+ returns: string
+ artifacts:
+ - "app_1/"
+
+ test:
+ type: "procedure"
+ handler: "b.test_procedure"
+ signature: ""
+ returns: string
+ artifacts:
+ - "app_2/"
+ meta:
+ use_mixins:
+ - "snowpark_shared"
+
+ hello_function:
+ type: "function"
+ handler: "c.hello_function"
+ signature:
+ - name: "name"
+ type: "string"
+ returns: string
+ artifacts:
+ - "c.py"
+ meta:
+ use_mixins:
+ - "snowpark_shared"

tests_integration/test_data/projects/snowpark_v2/snowflake.yml

@@ -16,28 +16,4 @@ entities:
  type: "string"
  returns: string
  artifacts:
- - "app_1/"
-
- test:
- type: "procedure"
- handler: "b.test_procedure"
- signature: ""
- returns: string
- artifacts:
- - "app_2/"
- meta:
- use_mixins:
- - "snowpark_shared"
-
- hello_function:
- type: "function"
- handler: "c.hello_function"
- signature:
- - name: "name"
- type: "string"
- returns: string
- artifacts:
- - "c.py"
- meta:
- use_mixins:
- - "snowpark_shared"
+ - "app_1/*"

tests_integration/test_snowpark.py

@@ -993,6 +993,31 @@ def test_snowpark_flow_v2(
  )
 
 
+@pytest.mark.integration
+def test_snowpark_with_glob_patterns(
+ _test_steps, project_directory, alter_snowflake_yml, test_database
+):
+ database = test_database.upper()
+ with project_directory("snowpark_glob_patterns") as tmp_dir:
+ _test_steps.snowpark_build_should_zip_files(
+ additional_files=[Path("app_1.zip"), Path("app_2.zip")]
+ )
+ _test_steps.snowpark_deploy_should_finish_successfully_and_return(
+ [
+ {
+ "object": f"{database}.PUBLIC.hello_procedure(name string)",
+ "status": "created",
+ "type": "procedure",
+ }
+ ]
+ )
+ _test_steps.snowpark_execute_should_return_expected_value(
+ object_type="procedure",
+ identifier="hello_procedure('foo')",
+ expected_value="Hello foo" + "Test procedure",
+ )
+
+
 @pytest.fixture
 def _test_setup(
  runner,