diff --git a/ingestion/src/metadata/ingestion/source/database/athena/client.py b/ingestion/src/metadata/ingestion/source/database/athena/client.py index 48764b987c61..b741266ece33 100644 --- a/ingestion/src/metadata/ingestion/source/database/athena/client.py +++ b/ingestion/src/metadata/ingestion/source/database/athena/client.py @@ -49,7 +49,7 @@ def get_database_tags(self, name: str) -> Optional[LFTags]: return lf_tags.LFTagOnDatabase except Exception as exc: logger.debug(traceback.format_exc()) - logger.error( + logger.warning( f"Unable to get lf tags for database resource [{name}] due to: {exc}" ) return None @@ -76,7 +76,7 @@ def get_table_and_column_tags( return LFTags(**response) except Exception as exc: logger.debug(traceback.format_exc()) - logger.error( + logger.warning( f"Unable to get lf tags for table resource [{table_name}] due to: {exc}" ) return None diff --git a/ingestion/tests/cli_e2e/test_cli_athena.py b/ingestion/tests/cli_e2e/test_cli_athena.py index 037854e95f3a..99a2f35f0494 100644 --- a/ingestion/tests/cli_e2e/test_cli_athena.py +++ b/ingestion/tests/cli_e2e/test_cli_athena.py @@ -17,6 +17,7 @@ import pytest +from metadata.ingestion.api.status import Status from metadata.workflow.metadata import MetadataWorkflow from .base.e2e_types import E2EType @@ -67,7 +68,7 @@ def view_column_lineage_count(self) -> int: @staticmethod def fqn_created_table() -> str: - return "e2e_athena.database_name.testdatalake_db.customers" + return "e2e_athena.database_name.e2e_db.customers" @staticmethod def fqn_deleted_table() -> None: @@ -75,7 +76,7 @@ def fqn_deleted_table() -> None: @staticmethod def get_includes_schemas() -> List[str]: - return ["testdatalake_db"] + return ["e2e_db"] @staticmethod def get_includes_tables() -> List[str]: @@ -87,7 +88,7 @@ def get_excludes_tables() -> List[str]: @staticmethod def expected_filtered_schema_includes() -> int: - return 4 + return 5 @staticmethod def expected_filtered_schema_excludes() -> int: @@ -95,15 +96,15 @@ def expected_filtered_schema_excludes() -> int: @staticmethod def expected_filtered_table_includes() -> int: - return 29 + return 7 @staticmethod def expected_filtered_table_excludes() -> int: - return 2 + return 7 @staticmethod def expected_filtered_mix() -> int: - return 6 + return 7 def retrieve_lineage(self, entity_fqn: str) -> dict: pass @@ -128,3 +129,20 @@ def test_create_table_with_profiler(self) -> None: result = self.run_command("profile") sink_status, source_status = self.retrieve_statuses(result) self.assert_for_table_with_profiler(source_status, sink_status) + + def assert_for_vanilla_ingestion( + self, source_status: Status, sink_status: Status + ) -> None: + self.assertTrue(len(source_status.failures) == 0) + self.assertTrue(len(source_status.warnings) == 0) + self.assertTrue(len(source_status.filtered) == 5) + self.assertTrue( + (len(source_status.records) + len(source_status.updated_records)) + >= self.expected_tables() + ) + self.assertTrue(len(sink_status.failures) == 0) + self.assertTrue(len(sink_status.warnings) == 0) + self.assertTrue( + (len(sink_status.records) + len(sink_status.updated_records)) + >= self.expected_tables() + )