Improvement : Custom Property search api

openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java

@@ -265,6 +265,11 @@ public static String customPropertyConfigError(String fieldName, String validati
     return String.format("Custom Property %s has invalid value %s", fieldName, validationMessages);
   }
 
+  public static String unknownCustomProperty(String propertyName, String entityType) {
+    return String.format(
+        "Custom property %s not found for entity type %s", propertyName, entityType);
+  }
+
   public static String invalidParent(Team parent, String child, TeamType childType) {
     return String.format(
         "Team %s of type %s can't be of parent of team %s of type %s",

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/Filter.java

@@ -21,6 +21,11 @@ public T addQueryParam(String name, Boolean value) {
     return (T) this;
   }
 
+  public T addQueryParam(String name, int value) {
+    queryParams.put(name, String.valueOf(value));
+    return (T) this;
+  }
+
   public void removeQueryParam(String name) {
     queryParams.remove(name);
   }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java

@@ -262,6 +262,18 @@ private void validateTableTypeConfig(CustomPropertyConfig config) {
     }
   }
 
+  public CustomProperty getCustomPropertyType(String entityType, String propertyName) {
+    EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties"));
+    Type typeEntity = getByName(null, entityType, fieldsParam, Include.ALL, false);
+    for (CustomProperty customProperty : typeEntity.getCustomProperties()) {
+      if (customProperty.getName().equals(propertyName)) {
+        return customProperty;
+      }
+    }
+    throw new IllegalArgumentException(
+        CatalogExceptionMessage.unknownCustomProperty(propertyName, entityType));
+  }
+
   /** Handles entity updated from PUT and POST operation. */
   public class TypeUpdater extends EntityUpdater {
     public TypeUpdater(Type original, Type updated, Operation operation) {

openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java

@@ -29,7 +29,9 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.UUID;
+import javax.validation.constraints.NotNull;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
@@ -42,12 +44,16 @@
 import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.schema.entity.type.CustomProperty;
 import org.openmetadata.schema.system.EventPublisherJob;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.service.Entity;
+import org.openmetadata.service.jdbi3.TypeRepository;
 import org.openmetadata.service.resources.Collection;
+import org.openmetadata.service.search.SearchListFilter;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.search.SearchRequest;
+import org.openmetadata.service.search.SearchSortFilter;
 import org.openmetadata.service.security.Authorizer;
 import org.openmetadata.service.security.policyevaluator.SubjectContext;
 import org.openmetadata.service.util.JsonUtils;
@@ -434,4 +440,89 @@ public Response reindexAllJobLastStatus(
     }
     return Response.status(Response.Status.NOT_FOUND).entity("No Last Run.").build();
   }
+
+  @GET
+  @Path("/customProperties")
+  @Operation(
+      operationId = "searchCustomProperties",
+      summary = "Search Custom Properties",
+      responses = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "search response",
+            content =
+                @Content(
+                    mediaType = "application/json",
+                    schema = @Schema(implementation = SearchResponse.class)))
+      })
+  public Response searchCustomProperties(
+      @Parameter(description = "Type of the entity", example = "table")
+          @QueryParam("entityType")
+          @NotNull(message = "entityType is required")
+          String entityType,
+      @Parameter(description = "Name of the custom property", example = "customPropertyName")
+          @QueryParam("propertyName")
+          @NotNull(message = "propertyName is required")
+          String propertyName,
+      @Parameter(
+              description = "search query term to search for custom properties",
+              schema = @Schema(type = "string"))
+          @QueryParam("q")
+          String q,
+      @Parameter(
+              description =
+                  "Elasticsearch query that will be combined with the query_string query generator from the `q` argument")
+          @QueryParam("query_filter")
+          String queryFilter,
+      @Parameter(description = "Starting point of the results", example = "20")
+          @QueryParam("from")
+          @DefaultValue("0")
+          int from,
+      @Parameter(description = "Number of results to return", example = "10")
+          @QueryParam("size")
+          @DefaultValue("10")
+          int size,
+      @Parameter(description = "ElasticSearch Index name, defaults to table_search_index")
+          @DefaultValue("dataAsset")
+          @QueryParam("index")
+          String index,
+      @Parameter(description = "Include deleted entities", example = "false")
+          @QueryParam("deleted")
+          @DefaultValue("false")
+          boolean deleted,
+      @Parameter(description = "Field to sort the results by", example = "_score")
+          @QueryParam("sort_field")
+          @DefaultValue("_score")
+          String sortField,
+      @Parameter(description = "Order to sort the results", example = "desc")
+          @QueryParam("sort_order")
+          @DefaultValue("desc")
+          String sortOrder,
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext) {
+
+    try {
+      SearchListFilter searchListFilter = new SearchListFilter();
+      searchListFilter.addQueryParam("entityType", entityType);
+      searchListFilter.addQueryParam("propertyName", propertyName);
+      searchListFilter.addQueryParam("index", index);
+      Optional.ofNullable(q).ifPresent(query -> searchListFilter.addQueryParam("query", query));
+      searchListFilter.addQueryParam("from", from);
+      searchListFilter.addQueryParam("size", size);
+      searchListFilter.addQueryParam("deleted", deleted);
+      TypeRepository typeRepository = (TypeRepository) Entity.getEntityRepository(Entity.TYPE);
+      CustomProperty property = typeRepository.getCustomPropertyType(entityType, propertyName);
+
+      SearchSortFilter searchSortFilter = new SearchSortFilter(sortField, sortOrder, null, null);
+
+      return Entity.getSearchRepository()
+          .searchCustomProperties(searchListFilter, searchSortFilter, queryFilter, property);
+
+    } catch (Exception e) {
+      LOG.error("Error searching custom properties: {}", e.getMessage(), e);
+      return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
+          .entity("Error searching custom properties. Exception: " + e.getMessage())
+          .build();
+    }
+  }
 }

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java

@@ -19,6 +19,7 @@
 import org.openmetadata.schema.dataInsight.DataInsightChartResult;
 import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart;
 import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList;
+import org.openmetadata.schema.entity.type.CustomProperty;
 import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
 import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.schema.tests.DataQualityReport;
@@ -198,6 +199,13 @@ Response searchEntityRelationship(
       String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted)
       throws IOException;
 
+  Response searchCustomProperties(
+      SearchListFilter searchListFilter,
+      SearchSortFilter searchSortFilter,
+      String queryFilter,
+      CustomProperty property)
+      throws IOException;
+
   Response searchDataQualityLineage(
       String fqn, int upstreamDepth, String queryFilter, boolean deleted) throws IOException;
 

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java

@@ -61,6 +61,7 @@
 import org.openmetadata.schema.analytics.ReportData;
 import org.openmetadata.schema.dataInsight.DataInsightChartResult;
 import org.openmetadata.schema.entity.classification.Tag;
+import org.openmetadata.schema.entity.type.CustomProperty;
 import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
 import org.openmetadata.schema.tests.DataQualityReport;
 import org.openmetadata.schema.tests.TestSuite;
@@ -944,6 +945,16 @@ public Response searchEntityRelationship(
         fqn, upstreamDepth, downstreamDepth, queryFilter, deleted);
   }
 
+  public Response searchCustomProperties(
+      SearchListFilter searchListFilter,
+      SearchSortFilter searchSortFilter,
+      String queryFilter,
+      CustomProperty property)
+      throws IOException {
+    return searchClient.searchCustomProperties(
+        searchListFilter, searchSortFilter, queryFilter, property);
+  }
+
   public Response searchDataQualityLineage(
       String fqn, int upstreamDepth, String queryFilter, boolean deleted) throws IOException {
     return searchClient.searchDataQualityLineage(fqn, upstreamDepth, queryFilter, deleted);

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java

@@ -150,10 +150,12 @@
 import org.openmetadata.schema.dataInsight.custom.FormulaHolder;
 import org.openmetadata.schema.entity.data.EntityHierarchy__1;
 import org.openmetadata.schema.entity.data.Table;
+import org.openmetadata.schema.entity.type.CustomProperty;
 import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
 import org.openmetadata.schema.tests.DataQualityReport;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
+import org.openmetadata.schema.type.customProperties.TableConfig;
 import org.openmetadata.sdk.exception.SearchException;
 import org.openmetadata.sdk.exception.SearchIndexNotFoundException;
 import org.openmetadata.service.Entity;
@@ -166,6 +168,7 @@
 import org.openmetadata.service.search.SearchAggregation;
 import org.openmetadata.service.search.SearchClient;
 import org.openmetadata.service.search.SearchIndexUtils;
+import org.openmetadata.service.search.SearchListFilter;
 import org.openmetadata.service.search.SearchRequest;
 import org.openmetadata.service.search.SearchSortFilter;
 import org.openmetadata.service.search.UpdateSearchEventsConstant;
@@ -944,6 +947,123 @@ public Response searchEntityRelationship(
     return Response.status(OK).entity(responseMap).build();
   }
 
+  public Response searchCustomProperties(
+      SearchListFilter searchListFilter,
+      SearchSortFilter searchSortFilter,
+      String queryFilter,
+      CustomProperty property)
+      throws IOException {
+
+    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+    String index = searchListFilter.getQueryParam("index");
+    searchSourceBuilder.from(Integer.parseInt(searchListFilter.getQueryParam("from")));
+    searchSourceBuilder.size(Integer.parseInt(searchListFilter.getQueryParam("size")));
+
+    String propertyName = searchListFilter.getQueryParam("propertyName");
+    String propertyType = property.getPropertyType().getName();
+    String extensionField = "extension." + propertyName;
+    String query = searchListFilter.getQueryParam("query");
+    BoolQueryBuilder mainQuery = QueryBuilders.boolQuery();
+
+    if (!nullOrEmpty(query)) {
+      switch (propertyType) {
+        case "enum",
+            "date-cp",
+            "dateTime-cp",
+            "markdown",
+            "sqlQuery",
+            "string",
+            "time-cp",
+            "integer",
+            "number",
+            "timestamp" -> mainQuery.must(QueryBuilders.matchQuery(extensionField, query));
+
+        case "duration" -> mainQuery.must(
+            QueryBuilders.matchPhrasePrefixQuery(extensionField, query));
+
+        case "email" -> mainQuery.must(
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.matchPhrasePrefixQuery(extensionField, query))
+                .should(QueryBuilders.matchQuery(extensionField, query))
+                .minimumShouldMatch(1));
+
+        case "entityReference", "entityReferenceList" -> mainQuery.must(
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.matchQuery(extensionField + ".displayName", query))
+                .should(QueryBuilders.matchQuery(extensionField + ".fullyQualifiedName", query))
+                .minimumShouldMatch(1));
+
+        case "timeInterval" -> mainQuery.must(
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.matchQuery(extensionField + ".start", query))
+                .should(QueryBuilders.matchQuery(extensionField + ".end", query))
+                .minimumShouldMatch(1));
+
+        case "table-cp" -> {
+          TableConfig tableConfig =
+              JsonUtils.convertValue(
+                  property.getCustomPropertyConfig().getConfig(), TableConfig.class);
+          BoolQueryBuilder tableQuery = QueryBuilders.boolQuery();
+          for (String column : tableConfig.getColumns()) {
+            tableQuery.should(
+                QueryBuilders.matchPhrasePrefixQuery(extensionField + ".rows." + column, query));
+          }
+          tableQuery.minimumShouldMatch(1);
+          mainQuery.must(tableQuery);
+        }
+
+        default -> mainQuery.must(QueryBuilders.existsQuery(extensionField));
+      }
+    } else {
+      mainQuery.must(QueryBuilders.existsQuery(extensionField));
+    }
+    mainQuery.must(QueryBuilders.termQuery("deleted", searchListFilter.getQueryParam("deleted")));
+
+    if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) {
+      try {
+        XContentParser filterParser =
+            XContentType.JSON
+                .xContent()
+                .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, queryFilter);
+        QueryBuilder filterQuery = SearchSourceBuilder.fromXContent(filterParser).query();
+
+        searchSourceBuilder.query(QueryBuilders.boolQuery().must(mainQuery).filter(filterQuery));
+
+      } catch (Exception ex) {
+        LOG.warn("Error parsing query_filter from query parameters, ignoring filter", ex);
+      }
+    } else {
+      searchSourceBuilder.query(mainQuery);
+    }
+
+    if (!nullOrEmpty(searchSortFilter.getSortField())) {
+      FieldSortBuilder fieldSortBuilder =
+          new FieldSortBuilder(searchSortFilter.getSortField())
+              .order(SortOrder.fromString(searchSortFilter.getSortType()));
+      if (!searchSortFilter.getSortField().equalsIgnoreCase("_score")) {
+        fieldSortBuilder.unmappedType("integer");
+      }
+      searchSourceBuilder.sort(fieldSortBuilder);
+    }
+
+    searchSourceBuilder.fetchSource(new FetchSourceContext(true, new String[] {}, new String[] {}));
+    searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS);
+    searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
+
+    try {
+      SearchResponse searchResponse =
+          client.search(
+              new es.org.elasticsearch.action.search.SearchRequest(
+                      Entity.getSearchRepository().getIndexOrAliasName(index))
+                  .source(searchSourceBuilder),
+              RequestOptions.DEFAULT);
+
+      return Response.status(OK).entity(searchResponse.toString()).build();
+    } catch (ElasticsearchStatusException e) {
+      throw new SearchException(String.format("Search failed due to %s", e.getMessage()));
+    }
+  }
+
   @Override
   public Response searchDataQualityLineage(
       String fqn, int upstreamDepth, String queryFilter, boolean deleted) throws IOException {