diff --git a/ingestion/src/metadata/applications/__init__.py b/ingestion/src/metadata/applications/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/ingestion/src/metadata/workflow/base.py b/ingestion/src/metadata/workflow/base.py index 32ec8455e19d..eb337e77c6f8 100644 --- a/ingestion/src/metadata/workflow/base.py +++ b/ingestion/src/metadata/workflow/base.py @@ -121,8 +121,6 @@ def stop(self) -> None: except Exception as exc: logger.warning(f"Error trying to close the step {step} due to [{exc}]") - self.source.close() - @property def timer(self) -> RepeatedTimer: """ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index 22ae5547bb7c..5ce5659c84d1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -188,6 +188,11 @@ public void execute(JobExecutionContext jobExecutionContext) { this.startApp(jobExecutionContext); } + @Override + public void configure() { + /* Not needed by default */ + } + public static AppRuntime getAppRuntime(App app) { return JsonUtils.convertValue(app.getRuntime(), ScheduledExecutionContext.class); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 62600d2d56a1..ffe9bb9c836d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -17,27 +17,24 @@ private ApplicationHandler() { public static void triggerApplicationOnDemand( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - // Native Application - try { - Class clz = Class.forName(app.getClassName()); - Object resource = clz.getConstructor().newInstance(); + runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand", "triggerOnDemand"); + } - // Call init Method - Method initMethod = resource.getClass().getMethod("init", App.class, CollectionDAO.class, SearchRepository.class); - initMethod.invoke(resource, app, daoCollection, searchRepository); + public static void scheduleApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) { + runMethodFromApplication(app, daoCollection, searchRepository, "scheduleInternal", "initializeExternalApp"); + } - // Call Trigger On Demand Method - Method triggerOnDemandMethod = resource.getClass().getMethod("triggerOnDemand"); - triggerOnDemandMethod.invoke(resource); - } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { - LOG.error("Exception encountered", e); - throw new RuntimeException(e); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } + public static void configureApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) { + runMethodFromApplication(app, daoCollection, searchRepository, "configure", "configure"); } - public static void scheduleApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) { + /** Load an App from its className and call its methods dynamically */ + public static void runMethodFromApplication( + App app, + CollectionDAO daoCollection, + SearchRepository searchRepository, + String internalMethodName, + String externalMethodName) { // Native Application try { Class clz = Class.forName(app.getClassName()); @@ -49,10 +46,10 @@ public static void scheduleApplication(App app, CollectionDAO daoCollection, Sea // Call Trigger On Demand Method if (app.getAppType() == AppType.Internal) { - Method scheduleMethod = resource.getClass().getMethod("scheduleInternal"); + Method scheduleMethod = resource.getClass().getMethod(internalMethodName); scheduleMethod.invoke(resource); } else if (app.getAppType() == AppType.External) { - Method scheduleMethod = resource.getClass().getMethod("initializeExternalApp"); + Method scheduleMethod = resource.getClass().getMethod(externalMethodName); scheduleMethod.invoke(resource); } } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java index 2083532f548d..c8f90ae2dfb5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java @@ -15,5 +15,7 @@ public interface NativeApplication extends Job { void initializeExternalApp(); + void configure(); + default void startApp(JobExecutionContext jobExecutionContext) {} } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index d8eeac68e66b..159aeca2c94f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -478,9 +478,14 @@ public Response create(@Context UriInfo uriInfo, @Context SecurityContext securi .getByName( uriInfo, create.getName(), new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()).build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.scheduleApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.configureApplication(app, Entity.getCollectionDAO(), searchRepository); } + // We don't want to store this information + app.setOpenMetadataServerConnection(null); return create(uriInfo, securityContext, app); } @@ -633,6 +638,37 @@ public Response scheduleApplication( throw new IllegalArgumentException("App is not of schedule type Scheduled."); } + @POST + @Path("/configure/{name}") + @Operation( + operationId = "configureApplication", + summary = "Configure an Application", + description = "Schedule a application to be run on demand.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Application", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Response.class))), + @ApiResponse(responseCode = "404", description = "Application for instance {id} is not found") + }) + public Response configureApplication( + @Context UriInfo uriInfo, + @Parameter(description = "Name of the App", schema = @Schema(type = "string")) @PathParam("name") String name, + @Context SecurityContext securityContext) { + App app = repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); + // The application will have the updated appConfiguration we can use to run the `configure` logic + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()).build()); + try { + ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository); + return Response.status(Response.Status.OK).entity("App has been configured.").build(); + } catch (RuntimeException e) { + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(String.format("Error configuring app [%s]", e.getMessage())) + .build(); + } + } + @POST @Path("/trigger/{name}") @Operation( diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json index ad056ba6fcee..64e015ce4961 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json @@ -4,6 +4,7 @@ "title": "AutoTaggerAppConfig", "description": "Configuration for the Auto Tagger External Application.", "type": "object", + "javaType": "org.openmetadata.schema.entity.app.external.AutoTaggerAppConfig", "definitions": { "autoTaggerApp": { "description": "Application type.", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json new file mode 100644 index 000000000000..ca6edd2023e3 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json @@ -0,0 +1,69 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/metaPilotAppConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MetaPilotAppConfig", + "description": "Configuration for the MetaPilot External Application.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppConfig", + "definitions": { + "metaPilotApp": { + "description": "Application type.", + "type": "string", + "enum": ["MetaPilot"], + "default": "MetaPilot" + }, + "serviceDatabases": { + "title": "Service Databases", + "description": "Choose the service and its databases you want to generate descriptions from.", + "type": "object", + "properties": { + "serviceName": { + "title": "Service Name", + "description": "Service Name to get descriptions from.", + "type": "string" + }, + "databases": { + "title": "Databases", + "description": "List of database names from the Service to get descriptions from.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": ["serviceName", "databases"] + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/metaPilotApp", + "default": "MetaPilot" + }, + "waiiInstance": { + "title": "WAII Instance", + "description": "WAII API host URL", + "type": "string", + "format": "URI", + "default": "https://tweakit.waii.ai/api/" + }, + "token": { + "title": "WAII API Token", + "description": "WAII API Token", + "type": "string", + "format": "password" + }, + "serviceDatabases": { + "title": "Service Databases", + "description": "Services and Databases configured to get the descriptions from.", + "type": "array", + "items": { + "$ref": "#/definitions/serviceDatabases" + } + } + }, + "additionalProperties": false, + "required": ["token"] +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json index f6398af87316..5f076d06d57c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json @@ -9,6 +9,9 @@ "oneOf": [ { "$ref": "./external/autoTaggerAppConfig.json" + }, + { + "$ref": "./external/metaPilotAppConfig.json" } ] }