Merge branch 'main' into alerts-refactoring-ui

ingestion/src/metadata/ingestion/source/database/athena/client.py

@@ -49,7 +49,7 @@ def get_database_tags(self, name: str) -> Optional[LFTags]:
             return lf_tags.LFTagOnDatabase
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
+            logger.warning(
                 f"Unable to get lf tags for database resource [{name}] due to: {exc}"
             )
         return None
@@ -76,7 +76,7 @@ def get_table_and_column_tags(
             return LFTags(**response)
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
+            logger.warning(
                 f"Unable to get lf tags for table resource [{table_name}] due to: {exc}"
             )
         return None

ingestion/tests/cli_e2e/test_cli_athena.py

@@ -17,6 +17,7 @@
 
 import pytest
 
+from metadata.ingestion.api.status import Status
 from metadata.workflow.metadata import MetadataWorkflow
 
 from .base.e2e_types import E2EType
@@ -67,15 +68,15 @@ def view_column_lineage_count(self) -> int:
 
     @staticmethod
     def fqn_created_table() -> str:
-        return "e2e_athena.database_name.testdatalake_db.customers"
+        return "e2e_athena.database_name.e2e_db.customers"
 
     @staticmethod
     def fqn_deleted_table() -> None:
         return None
 
     @staticmethod
     def get_includes_schemas() -> List[str]:
-        return ["testdatalake_db"]
+        return ["e2e_db"]
 
     @staticmethod
     def get_includes_tables() -> List[str]:
@@ -87,23 +88,23 @@ def get_excludes_tables() -> List[str]:
 
     @staticmethod
     def expected_filtered_schema_includes() -> int:
-        return 4
+        return 5
 
     @staticmethod
     def expected_filtered_schema_excludes() -> int:
         return 1
 
     @staticmethod
     def expected_filtered_table_includes() -> int:
-        return 29
+        return 7
 
     @staticmethod
     def expected_filtered_table_excludes() -> int:
-        return 2
+        return 7
 
     @staticmethod
     def expected_filtered_mix() -> int:
-        return 6
+        return 7
 
     def retrieve_lineage(self, entity_fqn: str) -> dict:
         pass
@@ -128,3 +129,20 @@ def test_create_table_with_profiler(self) -> None:
         result = self.run_command("profile")
         sink_status, source_status = self.retrieve_statuses(result)
         self.assert_for_table_with_profiler(source_status, sink_status)
+
+    def assert_for_vanilla_ingestion(
+        self, source_status: Status, sink_status: Status
+    ) -> None:
+        self.assertTrue(len(source_status.failures) == 0)
+        self.assertTrue(len(source_status.warnings) == 0)
+        self.assertTrue(len(source_status.filtered) == 5)
+        self.assertTrue(
+            (len(source_status.records) + len(source_status.updated_records))
+            >= self.expected_tables()
+        )
+        self.assertTrue(len(sink_status.failures) == 0)
+        self.assertTrue(len(sink_status.warnings) == 0)
+        self.assertTrue(
+            (len(sink_status.records) + len(sink_status.updated_records))
+            >= self.expected_tables()
+        )