From c2fbf892094eda1d24a7dd20f907f402d10cb582 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 21 Feb 2024 12:50:44 +0100 Subject: [PATCH 1/6] MINOR - Prepare App Exception & fix bigtable json --- .../service/apps/AppException.java | 32 +++++++++++++++++++ .../testConnections/database/bigtable.json | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java new file mode 100644 index 000000000000..dc8dcba82760 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java @@ -0,0 +1,32 @@ +package org.openmetadata.service.apps; + +import javax.ws.rs.core.Response; +import org.openmetadata.sdk.exception.WebServiceException; + +public class AppException extends WebServiceException { + + private static final String BY_NAME_MESSAGE = "Application [%s] Exception [%s] due to [%s]."; + private static final String ERROR_TYPE = "PIPELINE_SERVICE_ERROR"; + + public AppException(String message) { + super(Response.Status.BAD_REQUEST, ERROR_TYPE, message); + } + + private AppException(Response.Status status, String message) { + super(status, ERROR_TYPE, message); + } + + public static AppException byMessage( + String appName, String name, String errorMessage, Response.Status status) { + return new AppException(status, buildMessageByName(appName, name, errorMessage)); + } + + public static AppException byMessage(String appName, String name, String errorMessage) { + return new AppException( + Response.Status.BAD_REQUEST, buildMessageByName(appName, name, errorMessage)); + } + + private static String buildMessageByName(String appName, String name, String errorMessage) { + return String.format(BY_NAME_MESSAGE, appName, name, errorMessage); + } +} diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json b/openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json index 86ca0bb17ac2..2de02892236d 100644 --- a/openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json +++ b/openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json @@ -20,7 +20,7 @@ { "name": "ReadRows", "description": "Validate that we can read rows with the given credentials.", - "errorMessage": "Failed to read rows from BigTable, please validate to the credentials of service account" + "errorMessage": "Failed to read rows from BigTable, please validate to the credentials of service account", "shortCircuit": true, "mandatory": true } From 7ec7981b00263015e2a0af81226292bb2577938c Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 21 Feb 2024 16:53:08 +0100 Subject: [PATCH 2/6] Do not init the app all the time and allow to fetch initialized resources --- .../service/apps/ApplicationHandler.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 35b3afa19c20..2be87c0a4569 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -2,6 +2,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.HashMap; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.exception.UnhandledServerException; @@ -11,6 +12,12 @@ @Slf4j public class ApplicationHandler { + private static HashMap instances = new HashMap<>(); + + public static Object getAppInstance(String className) { + return instances.get(className); + } + private ApplicationHandler() { /*Helper*/ } @@ -30,20 +37,36 @@ public static void configureApplication( runMethodFromApplication(app, daoCollection, searchRepository, "configure"); } + private static Object runAppInit( + App app, CollectionDAO daoCollection, SearchRepository searchRepository) + throws ClassNotFoundException, + NoSuchMethodException, + InvocationTargetException, + InstantiationException, + IllegalAccessException { + Class clz = Class.forName(app.getClassName()); + Object resource = clz.getConstructor().newInstance(); + + // Call init Method + Method initMethod = + resource + .getClass() + .getMethod("init", App.class, CollectionDAO.class, SearchRepository.class); + initMethod.invoke(resource, app, daoCollection, searchRepository); + + return resource; + } + /** Load an App from its className and call its methods dynamically */ public static void runMethodFromApplication( App app, CollectionDAO daoCollection, SearchRepository searchRepository, String methodName) { // Native Application try { - Class clz = Class.forName(app.getClassName()); - Object resource = clz.getConstructor().newInstance(); - - // Call init Method - Method initMethod = - resource - .getClass() - .getMethod("init", App.class, CollectionDAO.class, SearchRepository.class); - initMethod.invoke(resource, app, daoCollection, searchRepository); + Object resource = getAppInstance(app.getClassName()); + if (resource == null) { + resource = runAppInit(app, daoCollection, searchRepository); + instances.put(app.getClassName(), resource); + } // Call method on demand Method scheduleMethod = resource.getClass().getMethod(methodName); From 546d8fd64314f4916c9f6a8790bd20d25900909f Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 22 Feb 2024 12:59:36 +0100 Subject: [PATCH 3/6] init installed apps --- .../org/openmetadata/service/apps/ApplicationHandler.java | 5 +++-- .../java/org/openmetadata/service/jdbi3/AppRepository.java | 6 +++--- .../openmetadata/service/resources/apps/AppResource.java | 5 +++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 39b2f8f13597..c70edba97d81 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -37,7 +37,7 @@ public static void configureApplication( runMethodFromApplication(app, daoCollection, searchRepository, "configure"); } - private static Object runAppInit( + public static Object runAppInit( App app, CollectionDAO daoCollection, SearchRepository searchRepository) throws ClassNotFoundException, NoSuchMethodException, @@ -53,6 +53,8 @@ private static Object runAppInit( Method initMethod = resource.getClass().getMethod("init", App.class); initMethod.invoke(resource, app); + instances.put(app.getClassName(), resource); + return resource; } @@ -64,7 +66,6 @@ public static void runMethodFromApplication( Object resource = getAppInstance(app.getClassName()); if (resource == null) { resource = runAppInit(app, daoCollection, searchRepository); - instances.put(app.getClassName(), resource); } // Call method on demand diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java index dfa42c3d6085..23d5d0ab7666 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java @@ -164,12 +164,12 @@ public void storeRelationships(App entity) { } } - public final List listAll() { + public final List listAll() { // forward scrolling, if after == null then first page is being asked List jsons = dao.listAfterWithOffset(Integer.MAX_VALUE, 0); - List entities = new ArrayList<>(); + List entities = new ArrayList<>(); for (String json : jsons) { - AppRunRecord entity = JsonUtils.readValue(json, AppRunRecord.class); + App entity = JsonUtils.readValue(json, App.class); entities.add(entity); } return entities; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 596fed2f7922..295fd16ebb8a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -143,6 +143,11 @@ public void initialize(OpenMetadataApplicationConfig config) { ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } } + + // Initialize installed applications + for (App app : repository.listAll()) { + ApplicationHandler.runAppInit(app, dao, searchRepository); + } } catch (Exception ex) { LOG.error("Failed in Create App Requests", ex); } From 24de920102d0994aaace01c69729128354bc4bb7 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 22 Feb 2024 14:07:49 +0100 Subject: [PATCH 4/6] Add runtime props --- .../org/openmetadata/service/resources/apps/AppResource.java | 1 + 1 file changed, 1 insertion(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 295fd16ebb8a..6a39de2bdd0a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -146,6 +146,7 @@ public void initialize(OpenMetadataApplicationConfig config) { // Initialize installed applications for (App app : repository.listAll()) { + setAppRuntimeProperties(app); ApplicationHandler.runAppInit(app, dao, searchRepository); } } catch (Exception ex) { From b7b35b6b39fff22b92ce41a2bdfebc43a7906456 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 22 Feb 2024 14:40:49 +0100 Subject: [PATCH 5/6] Add runtime props --- .../service/resources/apps/AppResource.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 6a39de2bdd0a..bca48931796b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -145,9 +145,15 @@ public void initialize(OpenMetadataApplicationConfig config) { } // Initialize installed applications - for (App app : repository.listAll()) { - setAppRuntimeProperties(app); - ApplicationHandler.runAppInit(app, dao, searchRepository); + for (App installedApp : repository.listAll()) { + App appWithBot = getAppForInit(installedApp.getName()); + if (appWithBot == null) { + LOG.error(String.format("Failed to init app [%s]. GET should return the installed app", installedApp.getName())); + } else { + setAppRuntimeProperties(appWithBot); + ApplicationHandler.runAppInit(appWithBot, dao, searchRepository); + LOG.info(String.format("Initialized installed app [%s]", installedApp.getName())); + } } } catch (Exception ex) { LOG.error("Failed in Create App Requests", ex); From f31ddf7accf914151e3cf021b8888e4d9f8edf5f Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 22 Feb 2024 14:49:15 +0100 Subject: [PATCH 6/6] Format --- .../org/openmetadata/service/resources/apps/AppResource.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index bca48931796b..3cf18bb2d130 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -148,7 +148,10 @@ public void initialize(OpenMetadataApplicationConfig config) { for (App installedApp : repository.listAll()) { App appWithBot = getAppForInit(installedApp.getName()); if (appWithBot == null) { - LOG.error(String.format("Failed to init app [%s]. GET should return the installed app", installedApp.getName())); + LOG.error( + String.format( + "Failed to init app [%s]. GET should return the installed app", + installedApp.getName())); } else { setAppRuntimeProperties(appWithBot); ApplicationHandler.runAppInit(appWithBot, dao, searchRepository);