diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java index 7935488bd8fd..8c59f8f25fb6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java @@ -265,6 +265,11 @@ public static String customPropertyConfigError(String fieldName, String validati return String.format("Custom Property %s has invalid value %s", fieldName, validationMessages); } + public static String unknownCustomProperty(String propertyName, String entityType) { + return String.format( + "Custom property %s not found for entity type %s", propertyName, entityType); + } + public static String invalidParent(Team parent, String child, TeamType childType) { return String.format( "Team %s of type %s can't be of parent of team %s of type %s", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/Filter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/Filter.java index d8de0a2fcf5a..20e843b21ac8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/Filter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/Filter.java @@ -21,6 +21,11 @@ public T addQueryParam(String name, Boolean value) { return (T) this; } + public T addQueryParam(String name, int value) { + queryParams.put(name, String.valueOf(value)); + return (T) this; + } + public void removeQueryParam(String name) { queryParams.remove(name); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java index 1d9078a264f6..0af51e808472 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java @@ -262,6 +262,18 @@ private void validateTableTypeConfig(CustomPropertyConfig config) { } } + public CustomProperty getCustomPropertyType(String entityType, String propertyName) { + EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties")); + Type typeEntity = getByName(null, entityType, fieldsParam, Include.ALL, false); + for (CustomProperty customProperty : typeEntity.getCustomProperties()) { + if (customProperty.getName().equals(propertyName)) { + return customProperty; + } + } + throw new IllegalArgumentException( + CatalogExceptionMessage.unknownCustomProperty(propertyName, entityType)); + } + /** Handles entity updated from PUT and POST operation. */ public class TypeUpdater extends EntityUpdater { public TypeUpdater(Type original, Type updated, Operation operation) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java index 8bc5338f26cc..62007d31e537 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java @@ -29,7 +29,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; +import javax.validation.constraints.NotNull; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -42,12 +44,16 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.system.EventPublisherJob; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.TypeRepository; import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.SearchRequest; +import org.openmetadata.service.search.SearchSortFilter; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.SubjectContext; import org.openmetadata.service.util.JsonUtils; @@ -434,4 +440,89 @@ public Response reindexAllJobLastStatus( } return Response.status(Response.Status.NOT_FOUND).entity("No Last Run.").build(); } + + @GET + @Path("/customProperties") + @Operation( + operationId = "searchCustomProperties", + summary = "Search Custom Properties", + responses = { + @ApiResponse( + responseCode = "200", + description = "search response", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SearchResponse.class))) + }) + public Response searchCustomProperties( + @Parameter(description = "Type of the entity", example = "table") + @QueryParam("entityType") + @NotNull(message = "entityType is required") + String entityType, + @Parameter(description = "Name of the custom property", example = "customPropertyName") + @QueryParam("propertyName") + @NotNull(message = "propertyName is required") + String propertyName, + @Parameter( + description = "search query term to search for custom properties", + schema = @Schema(type = "string")) + @QueryParam("q") + String q, + @Parameter( + description = + "Elasticsearch query that will be combined with the query_string query generator from the `q` argument") + @QueryParam("query_filter") + String queryFilter, + @Parameter(description = "Starting point of the results", example = "20") + @QueryParam("from") + @DefaultValue("0") + int from, + @Parameter(description = "Number of results to return", example = "10") + @QueryParam("size") + @DefaultValue("10") + int size, + @Parameter(description = "ElasticSearch Index name, defaults to table_search_index") + @DefaultValue("dataAsset") + @QueryParam("index") + String index, + @Parameter(description = "Include deleted entities", example = "false") + @QueryParam("deleted") + @DefaultValue("false") + boolean deleted, + @Parameter(description = "Field to sort the results by", example = "_score") + @QueryParam("sort_field") + @DefaultValue("_score") + String sortField, + @Parameter(description = "Order to sort the results", example = "desc") + @QueryParam("sort_order") + @DefaultValue("desc") + String sortOrder, + @Context UriInfo uriInfo, + @Context SecurityContext securityContext) { + + try { + SearchListFilter searchListFilter = new SearchListFilter(); + searchListFilter.addQueryParam("entityType", entityType); + searchListFilter.addQueryParam("propertyName", propertyName); + searchListFilter.addQueryParam("index", index); + Optional.ofNullable(q).ifPresent(query -> searchListFilter.addQueryParam("query", query)); + searchListFilter.addQueryParam("from", from); + searchListFilter.addQueryParam("size", size); + searchListFilter.addQueryParam("deleted", deleted); + TypeRepository typeRepository = (TypeRepository) Entity.getEntityRepository(Entity.TYPE); + CustomProperty property = typeRepository.getCustomPropertyType(entityType, propertyName); + + SearchSortFilter searchSortFilter = new SearchSortFilter(sortField, sortOrder, null, null); + + return Entity.getSearchRepository() + .searchCustomProperties(searchListFilter, searchSortFilter, queryFilter, property); + + } catch (Exception e) { + LOG.error("Error searching custom properties: {}", e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity("Error searching custom properties. Exception: " + e.getMessage()) + .build(); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index a02aaad5168e..1734c1b8f24a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -19,6 +19,7 @@ import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList; +import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.schema.tests.DataQualityReport; @@ -198,6 +199,13 @@ Response searchEntityRelationship( String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) throws IOException; + Response searchCustomProperties( + SearchListFilter searchListFilter, + SearchSortFilter searchSortFilter, + String queryFilter, + CustomProperty property) + throws IOException; + Response searchDataQualityLineage( String fqn, int upstreamDepth, String queryFilter, boolean deleted) throws IOException; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index d1e368fdc463..db7364e75a1b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -61,6 +61,7 @@ import org.openmetadata.schema.analytics.ReportData; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.entity.classification.Tag; +import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.tests.DataQualityReport; import org.openmetadata.schema.tests.TestSuite; @@ -944,6 +945,16 @@ public Response searchEntityRelationship( fqn, upstreamDepth, downstreamDepth, queryFilter, deleted); } + public Response searchCustomProperties( + SearchListFilter searchListFilter, + SearchSortFilter searchSortFilter, + String queryFilter, + CustomProperty property) + throws IOException { + return searchClient.searchCustomProperties( + searchListFilter, searchSortFilter, queryFilter, property); + } + public Response searchDataQualityLineage( String fqn, int upstreamDepth, String queryFilter, boolean deleted) throws IOException { return searchClient.searchDataQualityLineage(fqn, upstreamDepth, queryFilter, deleted); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index fec119ce0a46..b43688a9a100 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -150,10 +150,12 @@ import org.openmetadata.schema.dataInsight.custom.FormulaHolder; import org.openmetadata.schema.entity.data.EntityHierarchy__1; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.tests.DataQualityReport; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.sdk.exception.SearchException; import org.openmetadata.sdk.exception.SearchIndexNotFoundException; import org.openmetadata.service.Entity; @@ -166,6 +168,7 @@ import org.openmetadata.service.search.SearchAggregation; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchIndexUtils; +import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.SearchSortFilter; import org.openmetadata.service.search.UpdateSearchEventsConstant; @@ -944,6 +947,123 @@ public Response searchEntityRelationship( return Response.status(OK).entity(responseMap).build(); } + public Response searchCustomProperties( + SearchListFilter searchListFilter, + SearchSortFilter searchSortFilter, + String queryFilter, + CustomProperty property) + throws IOException { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + String index = searchListFilter.getQueryParam("index"); + searchSourceBuilder.from(Integer.parseInt(searchListFilter.getQueryParam("from"))); + searchSourceBuilder.size(Integer.parseInt(searchListFilter.getQueryParam("size"))); + + String propertyName = searchListFilter.getQueryParam("propertyName"); + String propertyType = property.getPropertyType().getName(); + String extensionField = "extension." + propertyName; + String query = searchListFilter.getQueryParam("query"); + BoolQueryBuilder mainQuery = QueryBuilders.boolQuery(); + + if (!nullOrEmpty(query)) { + switch (propertyType) { + case "enum", + "date-cp", + "dateTime-cp", + "markdown", + "sqlQuery", + "string", + "time-cp", + "integer", + "number", + "timestamp" -> mainQuery.must(QueryBuilders.matchQuery(extensionField, query)); + + case "duration" -> mainQuery.must( + QueryBuilders.matchPhrasePrefixQuery(extensionField, query)); + + case "email" -> mainQuery.must( + QueryBuilders.boolQuery() + .should(QueryBuilders.matchPhrasePrefixQuery(extensionField, query)) + .should(QueryBuilders.matchQuery(extensionField, query)) + .minimumShouldMatch(1)); + + case "entityReference", "entityReferenceList" -> mainQuery.must( + QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery(extensionField + ".displayName", query)) + .should(QueryBuilders.matchQuery(extensionField + ".fullyQualifiedName", query)) + .minimumShouldMatch(1)); + + case "timeInterval" -> mainQuery.must( + QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery(extensionField + ".start", query)) + .should(QueryBuilders.matchQuery(extensionField + ".end", query)) + .minimumShouldMatch(1)); + + case "table-cp" -> { + TableConfig tableConfig = + JsonUtils.convertValue( + property.getCustomPropertyConfig().getConfig(), TableConfig.class); + BoolQueryBuilder tableQuery = QueryBuilders.boolQuery(); + for (String column : tableConfig.getColumns()) { + tableQuery.should( + QueryBuilders.matchPhrasePrefixQuery(extensionField + ".rows." + column, query)); + } + tableQuery.minimumShouldMatch(1); + mainQuery.must(tableQuery); + } + + default -> mainQuery.must(QueryBuilders.existsQuery(extensionField)); + } + } else { + mainQuery.must(QueryBuilders.existsQuery(extensionField)); + } + mainQuery.must(QueryBuilders.termQuery("deleted", searchListFilter.getQueryParam("deleted"))); + + if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { + try { + XContentParser filterParser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, queryFilter); + QueryBuilder filterQuery = SearchSourceBuilder.fromXContent(filterParser).query(); + + searchSourceBuilder.query(QueryBuilders.boolQuery().must(mainQuery).filter(filterQuery)); + + } catch (Exception ex) { + LOG.warn("Error parsing query_filter from query parameters, ignoring filter", ex); + } + } else { + searchSourceBuilder.query(mainQuery); + } + + if (!nullOrEmpty(searchSortFilter.getSortField())) { + FieldSortBuilder fieldSortBuilder = + new FieldSortBuilder(searchSortFilter.getSortField()) + .order(SortOrder.fromString(searchSortFilter.getSortType())); + if (!searchSortFilter.getSortField().equalsIgnoreCase("_score")) { + fieldSortBuilder.unmappedType("integer"); + } + searchSourceBuilder.sort(fieldSortBuilder); + } + + searchSourceBuilder.fetchSource(new FetchSourceContext(true, new String[] {}, new String[] {})); + searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS); + searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS)); + + try { + SearchResponse searchResponse = + client.search( + new es.org.elasticsearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(index)) + .source(searchSourceBuilder), + RequestOptions.DEFAULT); + + return Response.status(OK).entity(searchResponse.toString()).build(); + } catch (ElasticsearchStatusException e) { + throw new SearchException(String.format("Search failed due to %s", e.getMessage())); + } + } + @Override public Response searchDataQualityLineage( String fqn, int upstreamDepth, String queryFilter, boolean deleted) throws IOException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 2ba538f37cdc..cebbaa8e49ba 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -70,10 +70,12 @@ import org.openmetadata.schema.dataInsight.custom.FormulaHolder; import org.openmetadata.schema.entity.data.EntityHierarchy__1; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.tests.DataQualityReport; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.sdk.exception.SearchException; import org.openmetadata.sdk.exception.SearchIndexNotFoundException; import org.openmetadata.service.Entity; @@ -86,6 +88,7 @@ import org.openmetadata.service.search.SearchAggregation; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchIndexUtils; +import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.SearchSortFilter; import org.openmetadata.service.search.indexes.APIEndpointIndex; @@ -946,6 +949,124 @@ public Response searchEntityRelationship( return Response.status(OK).entity(responseMap).build(); } + public Response searchCustomProperties( + SearchListFilter searchListFilter, + SearchSortFilter searchSortFilter, + String queryFilter, + CustomProperty property) + throws IOException { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + String index = searchListFilter.getQueryParam("index"); + searchSourceBuilder.from(Integer.parseInt(searchListFilter.getQueryParam("from"))); + searchSourceBuilder.size(Integer.parseInt(searchListFilter.getQueryParam("size"))); + + String propertyName = searchListFilter.getQueryParam("propertyName"); + String propertyType = property.getPropertyType().getName(); + String extensionField = "extension." + propertyName; + String query = searchListFilter.getQueryParam("query"); + BoolQueryBuilder mainQuery = QueryBuilders.boolQuery(); + + if (!nullOrEmpty(query)) { + switch (propertyType) { + case "enum", + "date-cp", + "dateTime-cp", + "markdown", + "sqlQuery", + "string", + "time-cp", + "integer", + "number", + "timestamp" -> mainQuery.must(QueryBuilders.matchQuery(extensionField, query)); + + case "duration" -> mainQuery.must( + QueryBuilders.matchPhrasePrefixQuery(extensionField, query)); + + case "email" -> mainQuery.must( + QueryBuilders.boolQuery() + .should(QueryBuilders.matchPhrasePrefixQuery(extensionField, query)) + .should(QueryBuilders.matchQuery(extensionField, query)) + .minimumShouldMatch(1)); + + case "entityReference", "entityReferenceList" -> mainQuery.must( + QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery(extensionField + ".displayName", query)) + .should(QueryBuilders.matchQuery(extensionField + ".fullyQualifiedName", query)) + .minimumShouldMatch(1)); + + case "timeInterval" -> mainQuery.must( + QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery(extensionField + ".start", query)) + .should(QueryBuilders.matchQuery(extensionField + ".end", query)) + .minimumShouldMatch(1)); + + case "table-cp" -> { + TableConfig tableConfig = + JsonUtils.convertValue( + property.getCustomPropertyConfig().getConfig(), TableConfig.class); + BoolQueryBuilder tableQuery = QueryBuilders.boolQuery(); + for (String column : tableConfig.getColumns()) { + tableQuery.should( + QueryBuilders.matchPhrasePrefixQuery(extensionField + ".rows." + column, query)); + } + tableQuery.minimumShouldMatch(1); + mainQuery.must(tableQuery); + } + + default -> mainQuery.must(QueryBuilders.existsQuery(extensionField)); + } + } else { + mainQuery.must(QueryBuilders.existsQuery(extensionField)); + } + + mainQuery.must(QueryBuilders.termQuery("deleted", searchListFilter.getQueryParam("deleted"))); + + if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { + try { + XContentParser filterParser = + XContentType.JSON + .xContent() + .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, queryFilter); + QueryBuilder filterQuery = SearchSourceBuilder.fromXContent(filterParser).query(); + + searchSourceBuilder.query(QueryBuilders.boolQuery().must(mainQuery).filter(filterQuery)); + + } catch (Exception ex) { + LOG.warn("Error parsing query_filter from query parameters, ignoring filter", ex); + } + } else { + searchSourceBuilder.query(mainQuery); + } + + if (!nullOrEmpty(searchSortFilter.getSortField())) { + FieldSortBuilder fieldSortBuilder = + new FieldSortBuilder(searchSortFilter.getSortField()) + .order(SortOrder.fromString(searchSortFilter.getSortType())); + if (!searchSortFilter.getSortField().equalsIgnoreCase("_score")) { + fieldSortBuilder.unmappedType("integer"); + } + searchSourceBuilder.sort(fieldSortBuilder); + } + + searchSourceBuilder.fetchSource(new FetchSourceContext(true, new String[] {}, new String[] {})); + searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS); + searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS)); + + try { + SearchResponse searchResponse = + client.search( + new os.org.opensearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(index)) + .source(searchSourceBuilder), + RequestOptions.DEFAULT); + + return Response.status(OK).entity(searchResponse.toString()).build(); + } catch (OpenSearchStatusException e) { + throw new SearchException(String.format("Search failed due to %s", e.getMessage())); + } + } + @Override public Response searchDataQualityLineage( String fqn, int upstreamDepth, String queryFilter, boolean deleted) throws IOException {