fix: test case reindex performance (#18723)

(cherry picked from commit dbd03c6)
open-metadata · Nov 21, 2024 · f4f0f9f · f4f0f9f
1 parent 2c85e7d
commit f4f0f9f
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 13 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
@@ -369,7 +369,7 @@ private void deleteAllTestCaseResults(String fqn) {
 
   @SneakyThrows
   private TestCaseResult getTestCaseResult(TestCase testCase) {
-    TestCaseResult testCaseResult;
+    TestCaseResult testCaseResult = null;
     if (testCase.getTestCaseResult() != null) {
       // we'll return the saved state if it exists otherwise we'll fetch it from the database
       // Should be the case if listing from the search repo. as the test case result
@@ -387,8 +387,6 @@ private TestCaseResult getTestCaseResult(TestCase testCase) {
       LOG.debug(
           "Error fetching test case result from search. Fetching from test case results from database",
           e);
-      testCaseResult =
-          timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
     }
     if (nullOrEmpty(testCaseResult)) {
       testCaseResult =

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
@@ -115,8 +115,11 @@ public TestSuiteRepository() {
   public void setFields(TestSuite entity, EntityUtil.Fields fields) {
     entity.setPipelines(
         fields.contains("pipelines") ? getIngestionPipelines(entity) : entity.getPipelines());
-    entity.withTests(fields.contains(UPDATE_FIELDS) ? getTestCases(entity) : entity.getTests());
-    entity.withTestCaseResultSummary(getResultSummary(entity.getId()));
+    entity.setTests(fields.contains(UPDATE_FIELDS) ? getTestCases(entity) : entity.getTests());
+    entity.setTestCaseResultSummary(
+        fields.contains("summary")
+            ? getResultSummary(entity.getId())
+            : entity.getTestCaseResultSummary());
     entity.setSummary(
         fields.contains("summary")
             ? getTestSummary(entity.getTestCaseResultSummary())
@@ -337,7 +340,7 @@ protected void postCreate(TestSuite entity) {
   @SneakyThrows
   private List<ResultSummary> getResultSummary(UUID testSuiteId) {
     List<ResultSummary> resultSummaries = new ArrayList<>();
-    ResultList<TestCaseResult> latestTestCaseResultResults;
+    ResultList<TestCaseResult> latestTestCaseResultResults = null;
     String groupBy = "testCaseFQN.keyword";
     SearchListFilter searchListFilter = new SearchListFilter();
     searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
@@ -348,16 +351,12 @@ private List<ResultSummary> getResultSummary(UUID testSuiteId) {
           entityTimeSeriesRepository.listLatestFromSearch(
               EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
     } catch (Exception e) {
-      // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
-      // to database
       LOG.debug(
           "Error fetching test case result from search. Fetching from test case results from database",
           e);
-      latestTestCaseResultResults =
-          entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
     }
 
-    if (nullOrEmpty(latestTestCaseResultResults.getData())) {
+    if (latestTestCaseResultResults == null || nullOrEmpty(latestTestCaseResultResults.getData())) {
       latestTestCaseResultResults =
           entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
     }

diff --git a/...ta-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java b/...ta-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java
@@ -42,7 +42,11 @@ public void removeNonIndexableFields(Map<String, Object> esDoc) {
   @Override
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> esDoc) {
     TestCase testCase =
-        Entity.getEntityByName(Entity.TEST_CASE, testCaseResult.getTestCaseFQN(), "*", Include.ALL);
+        Entity.getEntityByName(
+            Entity.TEST_CASE,
+            testCaseResult.getTestCaseFQN(),
+            "owners,testSuites,testDefinition,domain,tags",
+            Include.ALL);
     TestDefinition testDefinition =
         Entity.getEntityByName(
             Entity.TEST_DEFINITION,
@@ -80,7 +84,12 @@ private void setParentRelationships(TestCase testCase, Map<String, Object> esDoc
 
   private void setTableEntityParentRelations(
       MessageParser.EntityLink entityLink, Map<String, Object> esDoc) {
-    Table table = Entity.getEntityByName(Entity.TABLE, entityLink.getEntityFQN(), "*", Include.ALL);
+    Table table =
+        Entity.getEntityByName(
+            Entity.TABLE,
+            entityLink.getEntityFQN(),
+            "owners,columns,tags,followers,schemaDefinition,dataModel,extension,domain,dataProducts",
+            Include.ALL);
     EntityReference databaseSchemaReference = table.getDatabaseSchema();
     EntityReference databaseReference = table.getDatabase();
     EntityReference serviceReference = table.getService();