Merge branch 'main' into fix-13822

open-metadata · Jan 19, 2024 · 9947ad3 · 9947ad3
2 parents ce3ffd9 + f3f73a3
commit 9947ad3
Show file tree

Hide file tree

Showing 40 changed files with 2,496 additions and 1,154 deletions.
diff --git a/Makefile b/Makefile
@@ -102,12 +102,19 @@ snyk-dependencies-report:  ## Uses Snyk CLI to validate the project dependencies
 	snyk container test postgres:latest $(SNYK_ARGS) --json > security-report/postgres-scan.json | true;
 	snyk container test docker.elastic.co/elasticsearch/elasticsearch:7.10.2 $(SNYK_ARGS) --json > security-report/es-scan.json | true;
 
+.PHONY: snyk-ingestion-base-slim-report
+snyk-ingestion-base-slim-report:
+	@echo "Validating Ingestion Slim Container"
+	docker build -t openmetadata-ingestion-base-slim:scan -f ingestion/operators/docker/Dockerfile --build-arg INGESTION_DEPENDENCY=slim .
+	snyk container test openmetadata-ingestion-base-slim:scan --file=ingestion/operators/docker/Dockerfile $(SNYK_ARGS) --json > security-report/ingestion-docker-base-slim-scan.json | true;
+
 .PHONY: snyk-report
 snyk-report:  ## Uses Snyk CLI to run a security scan of the different pieces of the code
 	@echo "To run this locally, make sure to install and authenticate using the Snyk CLI: https://docs.snyk.io/snyk-cli/install-the-snyk-cli"
 	rm -rf security-report
 	mkdir -p security-report
 	$(MAKE) snyk-ingestion-report
+	$(MAKE) snyk-ingestion-base-slim-report
 	$(MAKE) snyk-airflow-apis-report
 	$(MAKE) snyk-server-report
 	$(MAKE) snyk-ui-report
@@ -126,7 +133,12 @@ export-snyk-pdf-report:  ## export json file from security-report/ to HTML
 .PHONY: build-ingestion-base-local
 build-ingestion-base-local:  ## Builds the ingestion DEV docker operator with the local ingestion files
 	$(MAKE) install_dev generate
-	docker build -f ingestion/operators/docker/Dockerfile-dev . -t openmetadata/ingestion-base:local
+	docker build -f ingestion/operators/docker/Dockerfile.ci . -t openmetadata/ingestion-base:local
+
+.PHONY: build-ingestion-base-slim-local
+build-ingestion-base-local:  ## Builds the ingestion DEV docker operator with the local ingestion files
+	$(MAKE) install_dev generate
+	docker build -f ingestion/operators/docker/Dockerfile.ci . -t openmetadata/ingestion-base-slim:local --build-arg INGESTION_DEPENDENCY=slim
 
 .PHONY: generate-schema-docs
 generate-schema-docs:  ## Generates markdown files for documenting the JSON Schemas

diff --git a/ingestion/operators/docker/Dockerfile-dev → ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile-dev → ingestion/operators/docker/Dockerfile.ci
diff --git a/ingestion/src/metadata/ingestion/source/database/athena/client.py b/ingestion/src/metadata/ingestion/source/database/athena/client.py
@@ -49,7 +49,7 @@ def get_database_tags(self, name: str) -> Optional[LFTags]:
             return lf_tags.LFTagOnDatabase
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
+            logger.warning(
                 f"Unable to get lf tags for database resource [{name}] due to: {exc}"
             )
         return None
@@ -76,7 +76,7 @@ def get_table_and_column_tags(
             return LFTags(**response)
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
+            logger.warning(
                 f"Unable to get lf tags for table resource [{table_name}] due to: {exc}"
             )
         return None
diff --git a/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py b/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py
@@ -37,9 +37,7 @@
 from metadata.profiler.metrics.static.mean import Mean
 from metadata.profiler.metrics.static.stddev import StdDev
 from metadata.profiler.metrics.static.sum import Sum
-from metadata.profiler.orm.functions.table_metric_construct import (
-    table_metric_construct_factory,
-)
+from metadata.profiler.orm.functions.table_metric_computer import TableMetricComputer
 from metadata.profiler.orm.registry import Dialects
 from metadata.profiler.processor.runner import QueryRunner
 from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
@@ -186,12 +184,13 @@ def _compute_table_metrics(
         # pylint: disable=protected-access
         try:
             dialect = runner._session.get_bind().dialect.name
-            row = table_metric_construct_factory.construct(
+            table_metric_computer: TableMetricComputer = TableMetricComputer(
                 dialect,
                 runner=runner,
                 metrics=metrics,
                 conn_config=self.service_connection_config,
             )
+            row = table_metric_computer.compute()
             if row:
                 return dict(row)
             return None