MINOR - Hide AutoTagger, add MetaPilot search, and fix apps status ca…

…lls (#14201) * MINOR - Hide AutoTagger & MetaPilot search & Fix apps status calls * Fix method call * Fix method call * Fix method call * "additionalProperties": true * Use EntityRef * Use ES sink * Update UI schema type
open-metadata · Jan 8, 2024 · e8eb2bd · e8eb2bd
1 parent c1d6a44
commit e8eb2bd
Show file tree

Hide file tree

Showing 10 changed files with 35 additions and 82 deletions.
diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py
@@ -133,7 +133,7 @@ def calculate_success(self) -> float:
         return self.runner.get_status().calculate_success()
 
     def get_failures(self) -> List[StackTraceError]:
-        return self.source.get_status().failures
+        return self.workflow_steps()[0].get_status().failures
 
     def workflow_steps(self) -> List[Step]:
         return [self.runner]

diff --git a/ingestion/src/metadata/workflow/workflow_output_handler.py b/ingestion/src/metadata/workflow/workflow_output_handler.py
@@ -28,8 +28,6 @@
     DEFAULT_EXAMPLE_FILE,
     EXAMPLES_WORKFLOW_PATH,
     WORKFLOW_FAILURE_MESSAGE,
-    WORKFLOW_SUCCESS_MESSAGE,
-    WORKFLOW_WARNING_MESSAGE,
     WorkflowType,
     print_error_msg,
     print_more_info,
@@ -122,12 +120,14 @@ def print_status(workflow: "IngestionWorkflow") -> None:
 
     print_workflow_summary(workflow)
 
-    if workflow.source.get_status().source_start_time:
+    # Get the time to execute the first step
+    first_step = workflow.workflow_steps()[0]
+    if first_step.get_status().source_start_time:
         log_ansi_encoded_string(
             color=ANSI.BRIGHT_CYAN,
             bold=True,
             message="Workflow finished in time: "
-            f"{pretty_print_time_duration(time.time()-workflow.source.get_status().source_start_time)}",
+            f"{pretty_print_time_duration(time.time()-first_step.get_status().source_start_time)}",
         )
 
     if workflow.result_status() == 1:
@@ -136,13 +136,3 @@ def print_status(workflow: "IngestionWorkflow") -> None:
             bold=True,
             message=WORKFLOW_FAILURE_MESSAGE,
         )
-    elif workflow.source.get_status().warnings or (
-        hasattr(workflow, "sink") and workflow.sink.get_status().warnings
-    ):
-        log_ansi_encoded_string(
-            color=ANSI.YELLOW, bold=True, message=WORKFLOW_WARNING_MESSAGE
-        )
-    else:
-        log_ansi_encoded_string(
-            color=ANSI.GREEN, bold=True, message=WORKFLOW_SUCCESS_MESSAGE
-        )
diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py
@@ -52,6 +52,7 @@
     PipelineType,
 )
 from metadata.generated.schema.metadataIngestion.applicationPipeline import (
+    ApplicationConfigType,
     ApplicationPipeline,
 )
 from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
@@ -376,7 +377,7 @@ def test_application_workflow(self):
             fullyQualifiedName="OpenMetadata.test_auto_tagger_application",
             sourceConfig=SourceConfig(
                 config=ApplicationPipeline(
-                    type="Application",
+                    type=ApplicationConfigType.Application,
                     appConfig=AppConfig(
                         __root__=AutoTaggerAppConfig(confidenceLevel=80)
                     ),

diff --git a/...service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/...service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
@@ -9,7 +9,7 @@
 import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig;
 import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
 import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
-import org.openmetadata.schema.metadataIngestion.DataInsightPipeline;
+import org.openmetadata.schema.metadataIngestion.MetadataToElasticSearchPipeline;
 import org.openmetadata.schema.metadataIngestion.SourceConfig;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.ProviderType;
@@ -98,7 +98,8 @@ private void createAndBindIngestionPipeline(
             .withDisplayName(INGESTION_PIPELINE_NAME)
             .withDescription(PIPELINE_DESCRIPTION)
             .withPipelineType(PipelineType.DATA_INSIGHT)
-            .withSourceConfig(new SourceConfig().withConfig(new DataInsightPipeline()))
+            // We're configuring Data Insights to use ES as a sink, so we apply ES in the config
+            .withSourceConfig(new SourceConfig().withConfig(new MetadataToElasticSearchPipeline()))
             .withAirflowConfig(
                 new AirflowConfig()
                     .withScheduleInterval(getCronMapper().map(quartzCron).asString()))

diff --git a/...-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json b/...-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json
diff --git a/.../resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/.../resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json
@@ -17,22 +17,28 @@
       "description": "Choose the service and its databases you want to generate descriptions from.",
       "type": "object",
       "properties": {
-        "serviceName": {
+        "service": {
           "title": "Service Name",
+          "placeholder": "Search Service",
           "description": "Service Name to get descriptions from.",
-          "type": "string"
+          "$ref": "../../../../type/entityReference.json",
+          "format": "autoComplete",
+          "autoCompleteType": "database_service_search_index"
         },
         "databases": {
           "title": "Databases",
           "description": "List of database names from the Service to get descriptions from.",
           "type": "array",
           "items": {
-            "type": "string"
+            "placeholder": "Search Databases",
+            "$ref": "../../../../type/entityReference.json",
+            "format": "autoComplete",
+            "autoCompleteType": "database_search_index"
           }
         }
       },
       "additionalProperties": false,
-      "required": ["serviceName", "databases"]
+      "required": ["service", "databases"]
     }
   },
   "properties": {

diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json
@@ -3,6 +3,7 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "OpenMetadataApplicationConfig",
   "description": "OpenMetadata Ingestion Framework definition for Applications, i.e., the YAML shape we require.",
+  "javaType": "org.openmetadata.schema.metadataIngestion.OpenMetadataAppConfig",
   "type": "object",
   "properties": {
     "workflowConfig": {

diff --git a/...metadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json b/...metadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json
@@ -17,22 +17,31 @@
       "description": "Choose the service and its databases you want to generate descriptions from.",
       "type": "object",
       "properties": {
-        "serviceName": {
+        "service": {
           "title": "Service Name",
+          "placeholder": "Search Service",
           "description": "Service Name to get descriptions from.",
-          "type": "string"
+          "type": "string",
+          "format": "autoComplete",
+          "autoCompleteType": "database_service_search_index",
+          "additionalProperties": true
         },
         "databases": {
           "title": "Databases",
           "description": "List of database names from the Service to get descriptions from.",
           "type": "array",
+          "additionalProperties": true,
           "items": {
-            "type": "string"
+            "placeholder": "Search Databases",
+            "type": "string",
+            "format": "autoComplete",
+            "autoCompleteType": "database_search_index",
+            "additionalProperties": true
           }
         }
       },
       "additionalProperties": false,
-      "required": ["serviceName", "databases"]
+      "required": ["service", "databases"]
     }
   },
   "properties": {

diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts b/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts
@@ -18,10 +18,10 @@ import enUS from '../../locale/languages/en-us.json';
 import esES from '../../locale/languages/es-es.json';
 import frFR from '../../locale/languages/fr-fr.json';
 import jaJP from '../../locale/languages/ja-jp.json';
+import nlNL from '../../locale/languages/nl-nl.json';
 import ptBR from '../../locale/languages/pt-br.json';
 import ruRU from '../../locale/languages/ru-ru.json';
 import zhCN from '../../locale/languages/zh-cn.json';
-import nlNL from '../../locale/languages/nl-nl.json';
 
 export enum SupportedLocales {
   English = 'en-US',