Issue-13782: RBAC for search

openmetadata-service/pom.xml

@@ -255,6 +255,10 @@
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-classic</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.jayway.jsonpath</groupId>
+      <artifactId>json-path</artifactId>
+    </dependency>
     <!-- Dependencies for secret store manager providers -->
     <dependency>
       <groupId>software.amazon.awssdk</groupId>

openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java

@@ -165,7 +165,7 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ
     jdbi = createAndSetupJDBI(environment, catalogConfig.getDataSourceFactory());
     Entity.setCollectionDAO(getDao(jdbi));
 
-    installSearchRepository(catalogConfig.getElasticSearchConfiguration());
+    initializeSearchRepository(catalogConfig.getElasticSearchConfiguration());
     // Initialize the MigrationValidationClient, used in the Settings Repository
     MigrationValidationClient.initialize(jdbi.onDemand(MigrationDAO.class), catalogConfig);
     // as first step register all the repositories
@@ -301,7 +301,7 @@ private void registerAuthServlets(OpenMetadataApplicationConfig config, Environm
     }
   }
 
-  protected void installSearchRepository(ElasticSearchConfiguration esConfig) {
+  protected void initializeSearchRepository(ElasticSearchConfiguration esConfig) {
     // initialize Search Repository, all repositories use SearchRepository this line should always
     // before initializing repository
     SearchRepository searchRepository = new SearchRepository(esConfig);

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java

@@ -451,7 +451,7 @@ protected Map<String, EntityReference> getGlossaryUsageFromES(String glossaryFqn
               .sortOrder("desc")
               .includeSourceFields(new ArrayList<>())
               .build();
-      Response response = searchRepository.search(searchRequest);
+      Response response = searchRepository.search(searchRequest, null);
       String json = (String) response.getEntity();
       Set<EntityReference> fqns = new TreeSet<>(compareEntityReferenceById);
       for (Iterator<JsonNode> it =

openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java

@@ -203,7 +203,7 @@ public Response search(
                 !subjectContext.isAdmin() && subjectContext.hasAnyRole(DOMAIN_ONLY_ACCESS_ROLE))
             .searchAfter(searchAfter)
             .build();
-    return searchRepository.search(request);
+    return searchRepository.search(request, subjectContext);
   }
 
   @GET

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java

@@ -22,14 +22,14 @@
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.service.exception.CustomExceptionMessage;
 import org.openmetadata.service.search.models.IndexMapping;
+import org.openmetadata.service.security.policyevaluator.SubjectContext;
 import org.openmetadata.service.util.SSLUtil;
 import os.org.opensearch.action.bulk.BulkRequest;
 import os.org.opensearch.action.bulk.BulkResponse;
 import os.org.opensearch.client.RequestOptions;
 
 public interface SearchClient {
   ExecutorService asyncExecutor = Executors.newFixedThreadPool(1);
-
   String UPDATE = "update";
 
   String ADD = "add";
@@ -106,7 +106,7 @@ public interface SearchClient {
 
   void createAliases(IndexMapping indexMapping);
 
-  Response search(SearchRequest request) throws IOException;
+  Response search(SearchRequest request, SubjectContext subjectContext) throws IOException;
 
   Response getDocByID(String indexName, String entityId) throws IOException;
 

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java

@@ -73,6 +73,7 @@
 import org.openmetadata.service.search.indexes.SearchIndex;
 import org.openmetadata.service.search.models.IndexMapping;
 import org.openmetadata.service.search.opensearch.OpenSearchClient;
+import org.openmetadata.service.security.policyevaluator.SubjectContext;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.workflows.searchIndex.ReindexingUtil;
 
@@ -723,8 +724,8 @@ public String getScriptWithParams(EntityInterface entity, Map<String, Object> fi
     return scriptTxt.toString();
   }
 
-  public Response search(SearchRequest request) throws IOException {
-    return searchClient.search(request);
+  public Response search(SearchRequest request, SubjectContext subjectContext) throws IOException {
+    return searchClient.search(request, subjectContext);
   }
 
   public Response getDocument(String indexName, UUID entityId) throws IOException {
@@ -838,7 +839,7 @@ public List<EntityReference> getEntitiesContainingFQNFromES(
               .build();
 
       // Execute the search and parse the response
-      Response response = search(searchRequest);
+      Response response = search(searchRequest, null);
       String json = (String) response.getEntity();
       Set<EntityReference> fqns = new TreeSet<>(compareEntityReferenceById);
 

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java

@@ -167,6 +167,8 @@
 import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchMostViewedEntitiesAggregator;
 import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchPageViewsByEntitiesAggregator;
 import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchUnusedAssetsAggregator;
+import org.openmetadata.service.search.elasticsearch.queries.ElasticQueryBuilder;
+import org.openmetadata.service.search.elasticsearch.queries.ElasticQueryBuilderFactory;
 import org.openmetadata.service.search.indexes.ContainerIndex;
 import org.openmetadata.service.search.indexes.DashboardDataModelIndex;
 import org.openmetadata.service.search.indexes.DashboardIndex;
@@ -187,17 +189,23 @@
 import org.openmetadata.service.search.indexes.TopicIndex;
 import org.openmetadata.service.search.indexes.UserIndex;
 import org.openmetadata.service.search.models.IndexMapping;
+import org.openmetadata.service.search.queries.OMQueryBuilder;
+import org.openmetadata.service.search.queries.QueryBuilderFactory;
+import org.openmetadata.service.search.security.RBACConditionEvaluator;
+import org.openmetadata.service.security.policyevaluator.SubjectContext;
 import org.openmetadata.service.util.FullyQualifiedName;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.workflows.searchIndex.ReindexingUtil;
 
 @Slf4j
-// Not tagged with Repository annotation as it is programmatically initialized
 public class ElasticSearchClient implements SearchClient {
 
   @SuppressWarnings("deprecated")
   protected final RestHighLevelClient client;
 
+  private final RBACConditionEvaluator rbacConditionEvaluator;
+  private final QueryBuilderFactory queryBuilderFactory;
+
   private final boolean isClientAvailable;
   public static final NamedXContentRegistry xContentRegistry;
 
@@ -224,6 +232,8 @@ public ElasticSearchClient(ElasticSearchConfiguration config) {
     client = createElasticSearchClient(config);
     clusterAlias = config != null ? config.getClusterAlias() : "";
     isClientAvailable = client != null;
+    queryBuilderFactory = new ElasticQueryBuilderFactory();
+    rbacConditionEvaluator = new RBACConditionEvaluator(queryBuilderFactory);
   }
 
   @Override
@@ -256,7 +266,6 @@ public void createIndex(IndexMapping indexMapping, String indexMappingContent) {
             "{} Created {}",
             indexMapping.getIndexName(clusterAlias),
             createIndexResponse.isAcknowledged());
-        // creating alias for indexes
         createAliases(indexMapping);
       } catch (Exception e) {
         LOG.error(
@@ -324,7 +333,7 @@ public void deleteIndex(IndexMapping indexMapping) {
   }
 
   @Override
-  public Response search(SearchRequest request) throws IOException {
+  public Response search(SearchRequest request, SubjectContext subjectContext) throws IOException {
     SearchSourceBuilder searchSourceBuilder =
         getSearchSourceBuilder(
             request.getIndex(), request.getQuery(), request.getFrom(), request.getSize());
@@ -387,7 +396,8 @@ public Response search(SearchRequest request) throws IOException {
         || request
             .getIndex()
             .equalsIgnoreCase(Entity.getSearchRepository().getIndexOrAliasName("dataAsset"))) {
-      BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+      es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder =
+          QueryBuilders.boolQuery();
       boolQueryBuilder.should(
           QueryBuilders.boolQuery()
               .must(searchSourceBuilder.query())
@@ -456,9 +466,6 @@ public Response search(SearchRequest request) throws IOException {
       searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));
 
       if (request.isGetHierarchy()) {
-        /*
-        Search for user input terms in name, fullyQualifiedName, displayName and glossary.fullyQualifiedName, glossary.displayName
-        */
         QueryBuilder baseQuery =
             QueryBuilders.boolQuery()
                 .should(searchSourceBuilder.query())
@@ -519,6 +526,8 @@ public Response search(SearchRequest request) throws IOException {
     }
 
     searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
+    buildSearchRBACQuery(subjectContext, searchSourceBuilder);
+
     try {
 
       SearchResponse searchResponse =
@@ -650,10 +659,11 @@ public SearchResultListMapper listWithOffset(
       try {
         XContentParser queryParser = createXContentParser(filter);
         XContentParser sourceParser = createXContentParser(filter);
-        QueryBuilder queryFromXContent = SearchSourceBuilder.fromXContent(queryParser).query();
+        es.org.elasticsearch.index.query.QueryBuilder queryFromXContent =
+            SearchSourceBuilder.fromXContent(queryParser).query();
         FetchSourceContext sourceFromXContent =
             SearchSourceBuilder.fromXContent(sourceParser).fetchSource();
-        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        es.org.elasticsearch.index.query.BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
         boolQuery =
             nullOrEmpty(q)
                 ? boolQuery.filter(queryFromXContent)
@@ -791,8 +801,9 @@ private void getLineage(
             XContentType.JSON
                 .xContent()
                 .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, queryFilter);
-        QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query();
-        BoolQueryBuilder newQuery =
+        es.org.elasticsearch.index.query.QueryBuilder filter =
+            SearchSourceBuilder.fromXContent(filterParser).query();
+        es.org.elasticsearch.index.query.BoolQueryBuilder newQuery =
             QueryBuilders.boolQuery().must(searchSourceBuilder.query()).filter(filter);
         searchSourceBuilder.query(newQuery);
       } catch (Exception ex) {
@@ -808,8 +819,8 @@ private void getLineage(
       tempMap.keySet().removeAll(FIELDS_TO_REMOVE);
       nodes.add(tempMap);
       for (Map<String, Object> lin : lineage) {
-        HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
-        HashMap<String, String> toEntity = (HashMap<String, String>) lin.get("toEntity");
+        Map<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
+        Map<String, String> toEntity = (HashMap<String, String>) lin.get("toEntity");
         if (direction.equalsIgnoreCase("lineage.fromEntity.fqnHash.keyword")) {
           if (!edges.contains(lin) && fromEntity.get("fqn").equals(fqn)) {
             edges.add(lin);
@@ -840,7 +851,7 @@ private Map<String, Object> searchPipelineLineage(
     es.org.elasticsearch.action.search.SearchRequest searchRequest =
         new es.org.elasticsearch.action.search.SearchRequest(
             Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
-    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+    es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
     boolQueryBuilder.should(
         QueryBuilders.boolQuery()
             .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn)));
@@ -858,8 +869,9 @@ private Map<String, Object> searchPipelineLineage(
             XContentType.JSON
                 .xContent()
                 .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, queryFilter);
-        QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query();
-        BoolQueryBuilder newQuery =
+        es.org.elasticsearch.index.query.QueryBuilder filter =
+            SearchSourceBuilder.fromXContent(filterParser).query();
+        es.org.elasticsearch.index.query.BoolQueryBuilder newQuery =
             QueryBuilders.boolQuery().must(searchSourceBuilder.query()).filter(filter);
         searchSourceBuilder.query(newQuery);
       } catch (Exception ex) {
@@ -960,9 +972,11 @@ public Response aggregate(String index, String fieldName, String value, String q
         XContentType.JSON
             .xContent()
             .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, query);
-    QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query();
+    es.org.elasticsearch.index.query.QueryBuilder filter =
+        SearchSourceBuilder.fromXContent(filterParser).query();
 
-    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(filter);
+    es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder =
+        QueryBuilders.boolQuery().must(filter);
     searchSourceBuilder
         .aggregation(
             AggregationBuilders.terms(fieldName)
@@ -1074,8 +1088,10 @@ public DataQualityReport genericAggregation(
           XContentType.JSON
               .xContent()
               .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, query);
-      QueryBuilder parsedQuery = SearchSourceBuilder.fromXContent(queryParser).query();
-      BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(parsedQuery);
+      es.org.elasticsearch.index.query.QueryBuilder parsedQuery =
+          SearchSourceBuilder.fromXContent(queryParser).query();
+      es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder =
+          QueryBuilders.boolQuery().must(parsedQuery);
       searchSourceBuilder.query(boolQueryBuilder);
     }
     searchSourceBuilder.size(0).timeout(new TimeValue(30, TimeUnit.SECONDS));
@@ -1112,8 +1128,10 @@ public JsonObject aggregate(String query, String index, JsonObject aggregationJs
           XContentType.JSON
               .xContent()
               .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, query);
-      QueryBuilder parsedQuery = SearchSourceBuilder.fromXContent(queryParser).query();
-      BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(parsedQuery);
+      es.org.elasticsearch.index.query.QueryBuilder parsedQuery =
+          SearchSourceBuilder.fromXContent(queryParser).query();
+      es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder =
+          QueryBuilders.boolQuery().must(parsedQuery);
       searchSourceBuilder.query(boolQueryBuilder);
     }
 
@@ -1553,7 +1571,10 @@ private static SearchSourceBuilder addAggregation(SearchSourceBuilder builder) {
   }
 
   private static SearchSourceBuilder searchBuilder(
-      QueryBuilder queryBuilder, HighlightBuilder hb, int from, int size) {
+      es.org.elasticsearch.index.query.QueryBuilder queryBuilder,
+      HighlightBuilder hb,
+      int from,
+      int size) {
     SearchSourceBuilder builder =
         new SearchSourceBuilder().query(queryBuilder).from(from).size(size);
     if (hb != null) {
@@ -1614,7 +1635,8 @@ public void deleteEntity(String indexName, String docId) {
   public void deleteEntityByFields(
       List<String> indexName, List<Pair<String, String>> fieldAndValue) {
     if (isClientAvailable) {
-      BoolQueryBuilder queryBuilder = new BoolQueryBuilder();
+      es.org.elasticsearch.index.query.BoolQueryBuilder queryBuilder =
+          new es.org.elasticsearch.index.query.BoolQueryBuilder();
       DeleteByQueryRequest deleteByQueryRequest =
           new DeleteByQueryRequest(indexName.toArray(new String[0]));
       for (Pair<String, String> p : fieldAndValue) {
@@ -1642,7 +1664,8 @@ public void softDeleteOrRestoreChildren(
     if (isClientAvailable) {
       UpdateByQueryRequest updateByQueryRequest =
           new UpdateByQueryRequest(indexName.toArray(new String[0]));
-      BoolQueryBuilder queryBuilder = new BoolQueryBuilder();
+      es.org.elasticsearch.index.query.BoolQueryBuilder queryBuilder =
+          new es.org.elasticsearch.index.query.BoolQueryBuilder();
       for (Pair<String, String> p : fieldAndValue) {
         queryBuilder.must(new TermQueryBuilder(p.getKey(), p.getValue()));
       }
@@ -1941,14 +1964,15 @@ private static SearchSourceBuilder buildQueryFilter(
       String dataInsightChartName) {
 
     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-    BoolQueryBuilder searchQueryFiler = new BoolQueryBuilder();
+    es.org.elasticsearch.index.query.BoolQueryBuilder searchQueryFiler =
+        new es.org.elasticsearch.index.query.BoolQueryBuilder();
 
     // Add team filter
     if (team != null
         && DataInsightChartRepository.SUPPORTS_TEAM_FILTER.contains(dataInsightChartName)) {
       List<String> teamArray = Arrays.asList(team.split("\\s*,\\s*"));
 
-      BoolQueryBuilder teamQueryFilter = QueryBuilders.boolQuery();
+      es.org.elasticsearch.index.query.BoolQueryBuilder teamQueryFilter = QueryBuilders.boolQuery();
       teamQueryFilter.should(
           QueryBuilders.termsQuery(DataInsightChartRepository.DATA_TEAM, teamArray));
       searchQueryFiler.must(teamQueryFilter);
@@ -1959,7 +1983,7 @@ private static SearchSourceBuilder buildQueryFilter(
         && DataInsightChartRepository.SUPPORTS_TIER_FILTER.contains(dataInsightChartName)) {
       List<String> tierArray = Arrays.asList(tier.split("\\s*,\\s*"));
 
-      BoolQueryBuilder tierQueryFilter = QueryBuilders.boolQuery();
+      es.org.elasticsearch.index.query.BoolQueryBuilder tierQueryFilter = QueryBuilders.boolQuery();
       tierQueryFilter.should(
           QueryBuilders.termsQuery(DataInsightChartRepository.DATA_ENTITY_TIER, tierArray));
       searchQueryFiler.must(tierQueryFilter);
@@ -1985,8 +2009,9 @@ private static SearchSourceBuilder buildQueryFilter(
             XContentType.JSON
                 .xContent()
                 .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, queryFilter);
-        QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query();
-        BoolQueryBuilder newQuery =
+        es.org.elasticsearch.index.query.QueryBuilder filter =
+            SearchSourceBuilder.fromXContent(filterParser).query();
+        es.org.elasticsearch.index.query.BoolQueryBuilder newQuery =
             QueryBuilders.boolQuery().must(searchSourceBuilder.query()).filter(filter);
         searchSourceBuilder.query(newQuery);
       } catch (Exception ex) {
@@ -2282,4 +2307,17 @@ private XContentParser createXContentParser(String query) throws IOException {
   public Object getLowLevelClient() {
     return client.getLowLevelClient();
   }
+
+  private void buildSearchRBACQuery(
+      SubjectContext subjectContext, SearchSourceBuilder searchSourceBuilder) {
+    if (subjectContext != null && !subjectContext.isAdmin()) {
+      if (rbacConditionEvaluator != null) {
+        OMQueryBuilder rbacQuery = rbacConditionEvaluator.evaluateConditions(subjectContext);
+        searchSourceBuilder.query(
+            QueryBuilders.boolQuery()
+                .must(searchSourceBuilder.query())
+                .filter(((ElasticQueryBuilder) rbacQuery).build()));
+      }
+    }
+  }
 }