Fix #11951 [backend] - Implement Profiler Global Config (#15889)

* feat: add global metric configuration for the profiler * style: ran java linting * fix: renamed disable to disabled * style: ran java linting * fix: json schema profiler config title
open-metadata · Apr 19, 2024 · d3b7de8 · d3b7de8
1 parent fd9e1c5
commit d3b7de8
Show file tree

Hide file tree

Showing 4 changed files with 150 additions and 2 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
@@ -56,6 +56,7 @@
 import org.openmetadata.schema.analytics.ReportData;
 import org.openmetadata.schema.analytics.WebAnalyticEvent;
 import org.openmetadata.schema.api.configuration.LoginConfiguration;
+import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration;
 import org.openmetadata.schema.auth.EmailVerificationToken;
 import org.openmetadata.schema.auth.PasswordResetToken;
 import org.openmetadata.schema.auth.PersonalAccessToken;
@@ -3950,6 +3951,7 @@ public static Settings getSettings(SettingsType configType, String json) {
             case SLACK_APP_CONFIGURATION -> JsonUtils.readValue(json, String.class);
             case SLACK_BOT, SLACK_INSTALLER -> JsonUtils.readValue(
                 json, new TypeReference<HashMap<String, Object>>() {});
+            case PROFILER_CONFIGURATION -> JsonUtils.readValue(json, ProfilerConfiguration.class);
             default -> throw new IllegalArgumentException("Invalid Settings Type " + configType);
           };
       settings.setConfigValue(value);

diff --git a/...a-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/...a-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java
@@ -10,6 +10,8 @@
 import io.dropwizard.jersey.validation.Validators;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
 import javax.validation.Validator;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response;
@@ -23,6 +25,8 @@
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.openmetadata.api.configuration.LogoConfiguration;
+import org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition;
+import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration;
 import org.openmetadata.schema.api.data.*;
 import org.openmetadata.schema.api.services.CreateDashboardService;
 import org.openmetadata.schema.api.services.CreateDatabaseService;
@@ -37,10 +41,12 @@
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
+import org.openmetadata.schema.profiler.MetricType;
 import org.openmetadata.schema.security.client.GoogleSSOClientConfig;
 import org.openmetadata.schema.settings.Settings;
 import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.schema.system.ValidationResponse;
+import org.openmetadata.schema.type.ColumnDataType;
 import org.openmetadata.schema.util.EntitiesCount;
 import org.openmetadata.schema.util.ServicesCount;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
@@ -310,6 +316,57 @@ void validate_test() throws HttpResponseException {
     Assertions.assertEquals(Boolean.TRUE, response.getMigrations().getPassed());
   }
 
+  @Test
+  void globalProfilerConfig(TestInfo test) throws HttpResponseException {
+    // Create a profiler config
+    ProfilerConfiguration profilerConfiguration = new ProfilerConfiguration();
+    MetricConfigurationDefinition intMetricConfigDefinition =
+        new MetricConfigurationDefinition()
+            .withDataType(ColumnDataType.INT)
+            .withMetrics(List.of(MetricType.COUNT, MetricType.FIRST_QUARTILE, MetricType.MEAN));
+    MetricConfigurationDefinition dateTimeMetricConfigDefinition =
+        new MetricConfigurationDefinition()
+            .withDataType(ColumnDataType.DATETIME)
+            .withDisabled(true);
+    profilerConfiguration.setMetricConfiguration(
+        List.of(intMetricConfigDefinition, dateTimeMetricConfigDefinition));
+    Settings profilerSettings =
+        new Settings()
+            .withConfigType(SettingsType.PROFILER_CONFIGURATION)
+            .withConfigValue(profilerConfiguration);
+    createSystemConfig(profilerSettings);
+    ProfilerConfiguration createdProfilerSettings =
+        JsonUtils.convertValue(
+            getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(),
+            ProfilerConfiguration.class);
+    Assertions.assertEquals(profilerConfiguration, createdProfilerSettings);
+
+    // Update the profiler config
+    profilerConfiguration.setMetricConfiguration(List.of(intMetricConfigDefinition));
+    profilerSettings =
+        new Settings()
+            .withConfigType(SettingsType.PROFILER_CONFIGURATION)
+            .withConfigValue(profilerConfiguration);
+    updateSystemConfig(profilerSettings);
+    ProfilerConfiguration updatedProfilerSettings =
+        JsonUtils.convertValue(
+            getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(),
+            ProfilerConfiguration.class);
+    Assertions.assertEquals(profilerConfiguration, updatedProfilerSettings);
+
+    // Delete the profiler config
+    profilerConfiguration.setMetricConfiguration(new ArrayList<>());
+    updateSystemConfig(
+        new Settings()
+            .withConfigType(SettingsType.PROFILER_CONFIGURATION)
+            .withConfigValue(profilerConfiguration));
+    updatedProfilerSettings =
+        JsonUtils.convertValue(
+            getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(),
+            ProfilerConfiguration.class);
+    Assertions.assertEquals(profilerConfiguration, updatedProfilerSettings);
+  }
+
   private static ValidationResponse getValidation() throws HttpResponseException {
     WebTarget target = getResource("system/status");
     return TestUtils.get(target, ValidationResponse.class, ADMIN_AUTH_HEADERS);
@@ -334,4 +391,9 @@ private static void updateSystemConfig(Settings updatedSetting) throws HttpRespo
     WebTarget target = getResource("system/settings");
     TestUtils.put(target, updatedSetting, Response.Status.OK, ADMIN_AUTH_HEADERS);
   }
+
+  private static void createSystemConfig(Settings updatedSetting) throws HttpResponseException {
+    WebTarget target = getResource("system/settings");
+    TestUtils.put(target, updatedSetting, Response.Status.CREATED, ADMIN_AUTH_HEADERS);
+  }
 }
diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/profilerConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/profilerConfiguration.json
@@ -0,0 +1,81 @@
+{
+  "$id": "https://open-metadata.org/schema/settings/profilerConfiguration.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "ProfilerConfiguration",
+  "description": "This schema defines the profiler configuration. It is used to configure globally the metrics to compute for specific data types.",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration",
+  "definitions": {
+    "metricType": {
+      "javaType": "org.openmetadata.schema.profiler.MetricType",
+      "description": "This schema defines all possible metric types in OpenMetadata.",
+      "type": "string",
+      "enum": [
+        "MEAN",
+        "COUNT",
+        "COUNT_IN_SET",
+        "COLUMN_COUNT",
+        "DISTINCT_COUNT",
+        "DISTINCT_RATIO",
+        "ILIKE_COUNT",
+        "LIKE_COUNT",
+        "NOT_LIKE_COUNT",
+        "REGEX_COUNT",
+        "NOT_REGEX_COUNT",
+        "MAX",
+        "MAX_LENGTH",
+        "MIN",
+        "MIN_LENGTH",
+        "NULL_COUNT",
+        "ROW_COUNT",
+        "STDDEV",
+        "SUM",
+        "UNIQUE_COUNT",
+        "UNIQUE_RATIO",
+        "COLUMN_NAMES",
+        "DUPLICATE_COUNT",
+        "ILIKE_RATIO",
+        "LIKE_RATIO",
+        "NULL_RATIO",
+        "IQR",
+        "NON_PARAMETRIC_SKEW",
+        "MEDIAN",
+        "FIRST_QUARTILE",
+        "THIRD_QUARTILE",
+        "SYSTEM",
+        "HISTOGRAM"
+      ]
+    },
+    "metricConfigurationDefinition": {
+      "type": "object",
+      "javaType": "org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition",
+      "description": "This schema defines the parameters that can be passed for a Test Case.",
+      "properties": {
+        "dataType": {
+          "$ref": "../entity/data/table.json#/definitions/dataType"
+        },
+        "metrics": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/metricType"
+          }
+        },
+        "disabled": {
+          "type": "boolean",
+          "description": "If true, the metric will not be computed for the data type.",
+          "default": false
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "properties": {
+      "metricConfiguration": {
+      "type": "array",
+      "items": {
+          "$ref": "#/definitions/metricConfigurationDefinition"
+      }
+    }
+  },
+  "additionalProperties": false
+}
diff --git a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json
@@ -27,7 +27,8 @@
         "loginConfiguration",
         "slackAppConfiguration",
         "slackBot",
-        "slackInstaller"
+        "slackInstaller",
+        "profilerConfiguration"
       ]
     }
   },
@@ -67,8 +68,10 @@
         },
         {
           "$ref": "../configuration/slackAppConfiguration.json"
+        },
+        {
+          "$ref": "../configuration/profilerConfiguration.json"
         }
-
       ]
     }
   },