Merge remote-tracking branch 'upstream/main' into MINOR-fix-incident-…

…filtering

ingestion/setup.py

@@ -117,7 +117,6 @@
     "sqlalchemy>=1.4.0,<2",
     "collate-sqllineage>=1.0.4",
     "tabulate==0.9.0",
-    "typing_extensions>=4.8.0",
     "typing-inspect",
     "wheel~=0.38.4",
 }

ingestion/src/metadata/profiler/interface/profiler_interface.py

@@ -18,7 +18,6 @@
 from typing import Any, Dict, List, Optional, Union
 
 from sqlalchemy import Column
-from typing_extensions import Self
 
 from metadata.generated.schema.entity.data.database import (
     Database,
@@ -155,7 +154,7 @@ def create(
         service_connection_config,
         ometa_client: Optional[OpenMetadata],
         **kwargs,
-    ) -> Self:
+    ) -> "ProfilerInterface":
         """create class method is used to dispatch the profiler protocol to the
         correct object based on the service connection object
 

ingestion/src/metadata/profiler/metrics/composed/null_ratio.py

@@ -51,7 +51,7 @@ def fn(self, res: Dict[str, Any]) -> Optional[float]:
         res_count = res.get(Count.name())
         res_null = res.get(NullCount.name())
 
-        if res_count and res_null is not None:
+        if res_count is not None and res_null is not None:
             return res_null / (res_null + res_count)
 
         return None

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java

@@ -247,26 +247,22 @@ private DataInsightDescriptionAndOwnerTemplate createDescriptionTemplate(
       double currentCompletedDescription = getCompletedDescriptionCount(last);
       double currentTotalCount = getTotalEntityFromDescriptionList(last);
 
-      // Calculate Percent Change
-      double previousDiff = previousTotalCount - previousCompletedDescription;
-      double currentDiff = currentTotalCount - currentCompletedDescription;
-
-      // Change
-      double percentChange = 0D;
-      if (previousDiff != 0) {
-        percentChange = ((currentDiff - previousDiff) / previousDiff) * 100;
+      // Previous Percent
+      double previousPercentCompleted = 0D;
+      if (previousTotalCount != 0) {
+        previousPercentCompleted = (previousCompletedDescription / previousTotalCount) * 100;
       }
-      // Completion
-      double percentCompleted = 0;
+      // Current Percent
+      double currentPercentCompleted = 0;
       if (currentTotalCount != 0) {
-        percentCompleted = (currentCompletedDescription / currentTotalCount) * 100;
+        currentPercentCompleted = (currentCompletedDescription / currentTotalCount) * 100;
       }
 
       return getTemplate(
           DataInsightDescriptionAndOwnerTemplate.MetricType.DESCRIPTION,
           PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE,
-          percentCompleted,
-          percentChange,
+          currentPercentCompleted,
+          currentPercentCompleted - previousPercentCompleted,
           numberOfDaysChange);
     }
 
@@ -300,27 +296,22 @@ private DataInsightDescriptionAndOwnerTemplate createOwnershipTemplate(
       double currentHasOwner = getCompletedOwnershipCount(last);
       double currentTotalCount = getTotalEntityFromOwnerList(last);
 
-      // Calculate Change
-      double previousDiff = previousTotalCount - previousHasOwner;
-      double currentDiff = currentTotalCount - currentHasOwner;
-
-      // Change Percent
-      double percentChange = 0D;
-      if (previousDiff != 0) {
-        percentChange = ((currentDiff - previousDiff) / previousDiff) * 100;
+      // Previous Percent
+      double previousPercentCompleted = 0D;
+      if (previousTotalCount != 0) {
+        previousPercentCompleted = (previousHasOwner / previousTotalCount) * 100;
       }
-
-      // Completion
-      double percentCompleted = 0;
+      // Current Percent
+      double currentPercentCompleted = 0;
       if (currentTotalCount != 0) {
-        percentCompleted = (currentHasOwner / currentTotalCount) * 100;
+        currentPercentCompleted = (currentHasOwner / currentTotalCount) * 100;
       }
 
       return getTemplate(
           DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER,
           PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE,
-          percentCompleted,
-          percentChange,
+          currentPercentCompleted,
+          currentPercentCompleted - previousPercentCompleted,
           numberOfDaysChange);
     }
 
@@ -479,28 +470,23 @@ private DataInsightDescriptionAndOwnerTemplate getTemplate(
   private long getTimeFromSchedule(
       AppSchedule appSchedule, JobExecutionContext jobExecutionContext) {
     AppSchedule.ScheduleTimeline timeline = appSchedule.getScheduleType();
-    switch (timeline) {
-      case HOURLY:
-        return 3600000L;
-      case DAILY:
-        return 86400000L;
-      case WEEKLY:
-        return 604800000L;
-      case MONTHLY:
-        return 2592000000L;
-      case CUSTOM:
+    return switch (timeline) {
+      case HOURLY -> 3600000L;
+      case DAILY -> 86400000L;
+      case WEEKLY -> 604800000L;
+      case MONTHLY -> 2592000000L;
+      case CUSTOM -> {
         if (jobExecutionContext.getTrigger() != null) {
           Trigger triggerQrz = jobExecutionContext.getTrigger();
           Date previousFire =
               triggerQrz.getPreviousFireTime() == null
                   ? triggerQrz.getStartTime()
                   : triggerQrz.getPreviousFireTime();
-          return previousFire.toInstant().toEpochMilli();
+          yield previousFire.toInstant().toEpochMilli();
         }
-        return 86400000L;
-      default:
-        throw new IllegalArgumentException("Invalid Trigger Type.");
-    }
+        yield 86400000L;
+      }
+    };
   }
 
   public static int getNumberOfDays(AppSchedule appSchedule) {

openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightDescriptionAndOwnerTemplate.java

@@ -28,9 +28,9 @@ public enum KpiCriteria {
     NOT_MET
   }
 
-  private final Double percentCompleted;
+  private final String percentCompleted;
   private boolean kpiAvailable;
-  private Double percentChange;
+  private String percentChange;
   private String targetKpi;
   private String numberOfDaysLeft;
   private String completeMessage;
@@ -47,9 +47,9 @@ public DataInsightDescriptionAndOwnerTemplate(
       String numberOfDaysLeft,
       int numberOfDaysChange,
       Map<String, Double> tierMap) {
-    this.percentCompleted = percentCompleted;
+    this.percentCompleted = String.format("%.2f", percentCompleted);
     this.targetKpi = targetKpi;
-    this.percentChange = percentChange;
+    this.percentChange = String.format("%.2f", percentChange);
     this.kpiAvailable = isKpiAvailable;
     this.numberOfDaysLeft = numberOfDaysLeft;
     this.tierMap = tierMap;
@@ -60,10 +60,10 @@ public DataInsightDescriptionAndOwnerTemplate(
     }
     this.completeMessage =
         String.format(
-            "The %s changed by <strong style=\"color: %s;\">%.2f%%</strong> per cent in the last week. %s",
+            "The %s changed by <strong style=\"color: %s;\">%s</strong>%% in the last week. %s",
             getMetricTypeMessage(metricType),
             color,
-            percentChange,
+            this.percentChange,
             getKpiCriteriaMessage(metricType, criteria));
   }
 
@@ -81,8 +81,8 @@ private String getKpiCriteriaMessage(MetricType metricType, KpiCriteria criteria
         return switch (criteria) {
           case MET -> "Great the Target Set for KPIs has been achieved. It's time to restructure your goals, set new KPIs and progress faster.";
           case IN_PROGRESS -> String.format(
-              "To meet the KPIs you will need a minimum of %s per cent completed description in the next %s days.",
-              targetKpi, numberOfDaysLeft);
+              "To meet the KPIs you will need a minimum of %s%% %s in the next %s days.",
+              targetKpi, getMetricTypeMessage(metricType).toLowerCase(), numberOfDaysLeft);
           case NOT_MET -> "The Target set for KPIs was not met it’s time to restructure your goals and progress faster.";
         };
       }
@@ -91,7 +91,7 @@ private String getKpiCriteriaMessage(MetricType metricType, KpiCriteria criteria
     return "";
   }
 
-  public Double getPercentCompleted() {
+  public String getPercentCompleted() {
     return percentCompleted;
   }
 
@@ -103,12 +103,12 @@ public void setTargetKpi(String targetKpi) {
     this.targetKpi = targetKpi;
   }
 
-  public Double getPercentChange() {
+  public String getPercentChange() {
     return percentChange;
   }
 
   public void setPercentChange(Double percentChange) {
-    this.percentChange = percentChange;
+    this.percentChange = String.format("%.2f", percentChange);
   }
 
   public boolean isKpiAvailable() {

openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightTotalAssetTemplate.java

@@ -14,41 +14,40 @@
 package org.openmetadata.service.events.scheduled.template;
 
 public class DataInsightTotalAssetTemplate {
-  private Double totalDataAssets;
-  private Double percentChangeTotalAssets;
+  private String totalDataAssets;
+  private String percentChangeTotalAssets;
   private String completeMessage;
-
   private int numberOfDaysChange;
 
   public DataInsightTotalAssetTemplate(
       Double totalDataAssets, Double percentChangeTotalAssets, int numberOfDaysChange) {
-    this.totalDataAssets = totalDataAssets;
-    this.percentChangeTotalAssets = percentChangeTotalAssets;
+    this.totalDataAssets = String.format("%.2f", totalDataAssets);
+    this.percentChangeTotalAssets = String.format("%.2f", percentChangeTotalAssets);
     this.numberOfDaysChange = numberOfDaysChange;
     String color = "#BF0000";
     if (percentChangeTotalAssets > 0) {
       color = "#008510";
     }
     completeMessage =
         String.format(
-            "In the past week, the Total Data Assets changed by a total of <span style=\"color: %s; font-weight: bold;\">%.2f%%</span>",
-            color, percentChangeTotalAssets);
+            "In the past week, the Total Data Assets changed by <span style=\"color: %s; font-weight: bold;\">%s</span>%%.",
+            color, this.percentChangeTotalAssets);
   }
 
-  public Double getTotalDataAssets() {
+  public String getTotalDataAssets() {
     return totalDataAssets;
   }
 
   public void setTotalDataAssets(Double totalDataAssets) {
-    this.totalDataAssets = totalDataAssets;
+    this.totalDataAssets = String.format("%.2f", totalDataAssets);
   }
 
-  public Double getPercentChangeTotalAssets() {
+  public String getPercentChangeTotalAssets() {
     return percentChangeTotalAssets;
   }
 
   public void setPercentChangeTotalAssets(Double percentChangeTotalAssets) {
-    this.percentChangeTotalAssets = percentChangeTotalAssets;
+    this.percentChangeTotalAssets = String.format("%.2f", percentChangeTotalAssets);
   }
 
   public String getCompleteMessage() {

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java

@@ -3708,14 +3708,6 @@ default String getTimeSeriesTableName() {
       return "data_quality_data_time_series";
     }
 
-    @SqlUpdate(
-        value =
-            "UPDATE data_quality_data_time_series SET incidentId = NULL "
-                + "WHERE entityFQNHash = :testCaseFQNHash and incidentId = :incidentStateId")
-    void cleanTestCaseIncident(
-        @BindFQN("testCaseFQNHash") String testCaseFQNHash,
-        @Bind("incidentStateId") String incidentStateId);
-
     @ConnectionAwareSqlUpdate(
         value =
             "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) "

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java

@@ -11,12 +11,12 @@
 import static org.openmetadata.service.util.RestUtil.ENTITY_NO_CHANGE;
 import static org.openmetadata.service.util.RestUtil.LOGICAL_TEST_CASES_ADDED;
 
+import com.google.common.collect.ImmutableSet;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.UUID;
 import javax.json.JsonPatch;
 import javax.ws.rs.core.Response;
@@ -258,6 +258,8 @@ public RestUtil.PutResponse<TestCaseResult> addTestCaseResult(
       String updatedBy, UriInfo uriInfo, String fqn, TestCaseResult testCaseResult) {
     // Validate the request content
     TestCase testCase = findByName(fqn, Include.NON_DELETED);
+    ArrayList<String> fields = new ArrayList<>();
+    fields.add(TEST_SUITE_FIELD);
 
     // set the test case resolution status reference if test failed, by either
     // creating a new incident or returning the stateId of an unresolved incident
@@ -269,6 +271,12 @@ public RestUtil.PutResponse<TestCaseResult> addTestCaseResult(
       // plotting the UI
       // even after the incident has been closed.
       testCaseResult.setIncidentId(incidentStateId);
+      // if the test case failed, we'll add the incidentId field to update the testCase entity on ln
+      // 293
+      fields.add(INCIDENTS_FIELD);
+    } else {
+      // If the test case passed, we'll remove the incidentId from the test case
+      testCase.setIncidentId(null);
     }
 
     // We add the incidentStateId in the DQ table to quickly link Test Case <> Incident
@@ -282,8 +290,7 @@ public RestUtil.PutResponse<TestCaseResult> addTestCaseResult(
             JsonUtils.pojoToJson(testCaseResult),
             incidentStateId != null ? incidentStateId.toString() : null);
 
-    setFieldsInternal(
-        testCase, new EntityUtil.Fields(allowedFields, Set.of(TEST_SUITE_FIELD, INCIDENTS_FIELD)));
+    setFieldsInternal(testCase, new EntityUtil.Fields(allowedFields, ImmutableSet.copyOf(fields)));
     setTestSuiteSummary(
         testCase, testCaseResult.getTimestamp(), testCaseResult.getTestCaseStatus(), false);
     setTestCaseResult(testCase, testCaseResult, false);

openmetadata-service/src/main/resources/json/data/app/DataInsightsReportApplication.json

@@ -6,7 +6,6 @@
     "sendToTeams": true
   },
   "appSchedule": {
-    "scheduleType": "Custom",
-    "cronExpression": "0 59 23 ? * 1"
+    "scheduleType": "Weekly"
   }
 }