App Resource - Add /configure endpoint & WAII app definition (#13983)

* App Configuration * TODO * Configure App with OM server info * Remove init * Rename configs * Remove extra source
open-metadata · Nov 17, 2023 · 6fb3f12 · 6fb3f12
1 parent 3cf2860
commit 6fb3f12
Show file tree

Hide file tree

Showing 9 changed files with 132 additions and 21 deletions.
diff --git a/ingestion/src/metadata/applications/__init__.py b/ingestion/src/metadata/applications/__init__.py
diff --git a/ingestion/src/metadata/workflow/base.py b/ingestion/src/metadata/workflow/base.py
@@ -121,8 +121,6 @@ def stop(self) -> None:
             except Exception as exc:
                 logger.warning(f"Error trying to close the step {step} due to [{exc}]")
 
-        self.source.close()
-
     @property
     def timer(self) -> RepeatedTimer:
         """

diff --git a/...tadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/...tadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java
@@ -188,6 +188,11 @@ public void execute(JobExecutionContext jobExecutionContext) {
     this.startApp(jobExecutionContext);
   }
 
+  @Override
+  public void configure() {
+    /* Not needed by default */
+  }
+
   public static AppRuntime getAppRuntime(App app) {
     return JsonUtils.convertValue(app.getRuntime(), ScheduledExecutionContext.class);
   }

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java
@@ -17,27 +17,24 @@ private ApplicationHandler() {
 
   public static void triggerApplicationOnDemand(
       App app, CollectionDAO daoCollection, SearchRepository searchRepository) {
-    // Native Application
-    try {
-      Class<?> clz = Class.forName(app.getClassName());
-      Object resource = clz.getConstructor().newInstance();
+    runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand", "triggerOnDemand");
+  }
 
-      // Call init Method
-      Method initMethod = resource.getClass().getMethod("init", App.class, CollectionDAO.class, SearchRepository.class);
-      initMethod.invoke(resource, app, daoCollection, searchRepository);
+  public static void scheduleApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) {
+    runMethodFromApplication(app, daoCollection, searchRepository, "scheduleInternal", "initializeExternalApp");
+  }
 
-      // Call Trigger On Demand Method
-      Method triggerOnDemandMethod = resource.getClass().getMethod("triggerOnDemand");
-      triggerOnDemandMethod.invoke(resource);
-    } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
-      LOG.error("Exception encountered", e);
-      throw new RuntimeException(e);
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException(e);
-    }
+  public static void configureApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) {
+    runMethodFromApplication(app, daoCollection, searchRepository, "configure", "configure");
   }
 
-  public static void scheduleApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) {
+  /** Load an App from its className and call its methods dynamically */
+  public static void runMethodFromApplication(
+      App app,
+      CollectionDAO daoCollection,
+      SearchRepository searchRepository,
+      String internalMethodName,
+      String externalMethodName) {
     // Native Application
     try {
       Class<?> clz = Class.forName(app.getClassName());
@@ -49,10 +46,10 @@ public static void scheduleApplication(App app, CollectionDAO daoCollection, Sea
 
       // Call Trigger On Demand Method
       if (app.getAppType() == AppType.Internal) {
-        Method scheduleMethod = resource.getClass().getMethod("scheduleInternal");
+        Method scheduleMethod = resource.getClass().getMethod(internalMethodName);
         scheduleMethod.invoke(resource);
       } else if (app.getAppType() == AppType.External) {
-        Method scheduleMethod = resource.getClass().getMethod("initializeExternalApp");
+        Method scheduleMethod = resource.getClass().getMethod(externalMethodName);
         scheduleMethod.invoke(resource);
       }
     } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java
@@ -15,5 +15,7 @@ public interface NativeApplication extends Job {
 
   void initializeExternalApp();
 
+  void configure();
+
   default void startApp(JobExecutionContext jobExecutionContext) {}
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java
@@ -478,9 +478,14 @@ public Response create(@Context UriInfo uriInfo, @Context SecurityContext securi
             .getByName(
                 uriInfo, create.getName(), new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields()));
     App app = getApplication(definition, create, securityContext.getUserPrincipal().getName());
+    app.setOpenMetadataServerConnection(
+        new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()).build());
     if (app.getScheduleType().equals(ScheduleType.Scheduled)) {
       ApplicationHandler.scheduleApplication(app, Entity.getCollectionDAO(), searchRepository);
+      ApplicationHandler.configureApplication(app, Entity.getCollectionDAO(), searchRepository);
     }
+    // We don't want to store this information
+    app.setOpenMetadataServerConnection(null);
     return create(uriInfo, securityContext, app);
   }
 
@@ -633,6 +638,37 @@ public Response scheduleApplication(
     throw new IllegalArgumentException("App is not of schedule type Scheduled.");
   }
 
+  @POST
+  @Path("/configure/{name}")
+  @Operation(
+      operationId = "configureApplication",
+      summary = "Configure an Application",
+      description = "Schedule a application to be run on demand.",
+      responses = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "The Application",
+            content = @Content(mediaType = "application/json", schema = @Schema(implementation = Response.class))),
+        @ApiResponse(responseCode = "404", description = "Application for instance {id} is not found")
+      })
+  public Response configureApplication(
+      @Context UriInfo uriInfo,
+      @Parameter(description = "Name of the App", schema = @Schema(type = "string")) @PathParam("name") String name,
+      @Context SecurityContext securityContext) {
+    App app = repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields()));
+    // The application will have the updated appConfiguration we can use to run the `configure` logic
+    app.setOpenMetadataServerConnection(
+        new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()).build());
+    try {
+      ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository);
+      return Response.status(Response.Status.OK).entity("App has been configured.").build();
+    } catch (RuntimeException e) {
+      return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
+          .entity(String.format("Error configuring app [%s]", e.getMessage()))
+          .build();
+    }
+  }
+
   @POST
   @Path("/trigger/{name}")
   @Operation(

diff --git a/...resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json b/...resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json
@@ -4,6 +4,7 @@
   "title": "AutoTaggerAppConfig",
   "description": "Configuration for the Auto Tagger External Application.",
   "type": "object",
+  "javaType": "org.openmetadata.schema.entity.app.external.AutoTaggerAppConfig",
   "definitions": {
     "autoTaggerApp": {
       "description": "Application type.",

diff --git a/.../resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/.../resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json
@@ -0,0 +1,69 @@
+{
+  "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/metaPilotAppConfig.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "MetaPilotAppConfig",
+  "description": "Configuration for the MetaPilot External Application.",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppConfig",
+  "definitions": {
+    "metaPilotApp": {
+      "description": "Application type.",
+      "type": "string",
+      "enum": ["MetaPilot"],
+      "default": "MetaPilot"
+    },
+    "serviceDatabases": {
+      "title": "Service Databases",
+      "description": "Choose the service and its databases you want to generate descriptions from.",
+      "type": "object",
+      "properties": {
+        "serviceName": {
+          "title": "Service Name",
+          "description": "Service Name to get descriptions from.",
+          "type": "string"
+        },
+        "databases": {
+          "title": "Databases",
+          "description": "List of database names from the Service to get descriptions from.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "additionalProperties": false,
+      "required": ["serviceName", "databases"]
+    }
+  },
+  "properties": {
+    "type": {
+      "title": "Service Type",
+      "description": "Service Type",
+      "$ref": "#/definitions/metaPilotApp",
+      "default": "MetaPilot"
+    },
+    "waiiInstance": {
+      "title": "WAII Instance",
+      "description": "WAII API host URL",
+      "type": "string",
+      "format": "URI",
+      "default": "https://tweakit.waii.ai/api/"
+    },
+    "token": {
+      "title": "WAII API Token",
+      "description": "WAII API Token",
+      "type": "string",
+      "format": "password"
+    },
+    "serviceDatabases": {
+      "title": "Service Databases",
+      "description": "Services and Databases configured to get the descriptions from.",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/serviceDatabases"
+      }
+    }
+  },
+  "additionalProperties": false,
+  "required": ["token"]
+}
diff --git a/...in/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json b/...in/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json
@@ -9,6 +9,9 @@
       "oneOf": [
         {
           "$ref": "./external/autoTaggerAppConfig.json"
+        },
+        {
+          "$ref": "./external/metaPilotAppConfig.json"
         }
       ]
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,6 +9,9 @@ @@
           "oneOf": [
             {
               "$ref": "./external/autoTaggerAppConfig.json"
+            },
+            {
+              "$ref": "./external/metaPilotAppConfig.json"
             }
           ]
         }
@@ Expand Down @@