feat(search): search access controls (datahub-project#9892)

Co-authored-by: Chris Collins <chriscollins3456@gmail.com>

build.gradle

@@ -17,7 +17,7 @@ buildscript {
 
   ext.javaClassVersion = { p ->
     // If Spring 6 is present, hard dependency on jdk17
-    if (p.configurations.any { it.getDependencies().any{
+    if (p.configurations.any { it.getDependencies().any {
       (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
               || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
     }}) {
@@ -43,7 +43,7 @@ buildscript {
   ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.3'
   ext.jettyVersion = '11.0.19'
-  ext.playVersion = '2.8.18'
+  ext.playVersion = '2.8.21'
   ext.log4jVersion = '2.19.0'
   ext.slf4jVersion = '1.7.36'
   ext.logbackClassic = '1.4.14'
@@ -132,7 +132,7 @@ project.ext.externalDependency = [
     'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:21.0',
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:7.0.0',
-    'guice4': 'com.google.inject:guice:4.2.3', // Used for frontend while still on old Play version
+    'guicePlay': 'com.google.inject:guice:5.0.1', // Used for frontend while still on old Play version
     'guava': 'com.google.guava:guava:32.1.2-jre',
     'h2': 'com.h2database:h2:2.2.224',
     'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',

datahub-frontend/app/auth/AuthModule.java

@@ -8,9 +8,11 @@
 import com.datahub.authentication.Actor;
 import com.datahub.authentication.ActorType;
 import com.datahub.authentication.Authentication;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.inject.AbstractModule;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
+import com.google.inject.name.Named;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.entity.client.SystemRestliEntityClient;
 import com.linkedin.metadata.restli.DefaultRestliClientFactory;
@@ -20,6 +22,13 @@
 import controllers.SsoCallbackController;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
+
+import io.datahubproject.metadata.context.ActorContext;
+import io.datahubproject.metadata.context.AuthorizerContext;
+import io.datahubproject.metadata.context.EntityRegistryContext;
+import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.metadata.context.OperationContextConfig;
+import io.datahubproject.metadata.context.SearchContext;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -152,6 +161,31 @@ protected Authentication provideSystemAuthentication() {
         Collections.emptyMap());
   }
 
+  @Provides
+  @Singleton
+  @Named("systemOperationContext")
+  protected OperationContext provideOperationContext(final Authentication systemAuthentication,
+                                                     final ConfigurationProvider configurationProvider) {
+    ActorContext systemActorContext =
+            ActorContext.builder()
+                    .systemAuth(true)
+                    .authentication(systemAuthentication)
+                    .build();
+    OperationContextConfig systemConfig = OperationContextConfig.builder()
+            .searchAuthorizationConfiguration(configurationProvider.getAuthorization().getSearch())
+            .allowSystemAuthentication(true)
+            .build();
+
+    return OperationContext.builder()
+            .operationContextConfig(systemConfig)
+            .systemActorContext(systemActorContext)
+            .searchContext(SearchContext.EMPTY)
+            .entityRegistryContext(EntityRegistryContext.EMPTY)
+            // Authorizer.EMPTY doesn't actually apply to system auth
+            .authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build())
+            .build(systemAuthentication);
+  }
+
   @Provides
   @Singleton
   protected ConfigurationProvider provideConfigurationProvider() {
@@ -163,13 +197,13 @@ protected ConfigurationProvider provideConfigurationProvider() {
   @Provides
   @Singleton
   protected SystemEntityClient provideEntityClient(
-      final Authentication systemAuthentication,
+      @Named("systemOperationContext") final OperationContext systemOperationContext,
       final ConfigurationProvider configurationProvider) {
     return new SystemRestliEntityClient(
+            systemOperationContext,
         buildRestliClient(),
         new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
         _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
-        systemAuthentication,
         configurationProvider.getCache().getClient().getEntityClient());
   }
 

datahub-frontend/app/config/ConfigurationProvider.java

@@ -1,5 +1,6 @@
 package config;
 
+import com.datahub.authorization.AuthorizationConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
 import com.linkedin.metadata.config.cache.CacheConfiguration;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
@@ -26,4 +27,7 @@ public class ConfigurationProvider {
 
   /** Configuration for the view layer */
   private VisualConfiguration visualConfig;
+
+  /** Configuration for authorization */
+  private AuthorizationConfiguration authorization;
 }

datahub-frontend/play.gradle

@@ -76,7 +76,7 @@ dependencies {
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
-  runtimeOnly externalDependency.guice4
+  runtimeOnly externalDependency.guicePlay
   runtimeOnly (externalDependency.playDocs) {
     exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
   }
@@ -90,7 +90,7 @@ dependencies {
 
 play {
   platform {
-    playVersion = '2.8.18'
+    playVersion = '2.8.21'
     scalaVersion = '2.12'
     javaVersion = JavaVersion.VERSION_11
   }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -4,7 +4,6 @@
 public class Constants {
 
   private Constants() {}
-  ;
 
   public static final String URN_FIELD_NAME = "urn";
   public static final String URNS_FIELD_NAME = "urns";

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -321,6 +321,7 @@
 import com.linkedin.datahub.graphql.types.ownership.OwnershipType;
 import com.linkedin.datahub.graphql.types.policy.DataHubPolicyType;
 import com.linkedin.datahub.graphql.types.query.QueryType;
+import com.linkedin.datahub.graphql.types.restricted.RestrictedType;
 import com.linkedin.datahub.graphql.types.role.DataHubRoleType;
 import com.linkedin.datahub.graphql.types.rolemetadata.RoleType;
 import com.linkedin.datahub.graphql.types.schemafield.SchemaFieldType;
@@ -349,6 +350,7 @@
 import com.linkedin.metadata.service.LineageService;
 import com.linkedin.metadata.service.OwnershipTypeService;
 import com.linkedin.metadata.service.QueryService;
+import com.linkedin.metadata.service.RestrictedService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
 import com.linkedin.metadata.timeline.TimelineService;
@@ -416,6 +418,7 @@ public class GmsGraphQLEngine {
   private final QueryService queryService;
   private final DataProductService dataProductService;
   private final FormService formService;
+  private final RestrictedService restrictedService;
 
   private final FeatureFlags featureFlags;
 
@@ -468,6 +471,7 @@ public class GmsGraphQLEngine {
   private final EntityTypeType entityTypeType;
   private final FormType formType;
   private final IncidentType incidentType;
+  private final RestrictedType restrictedType;
 
   private final int graphQLQueryComplexityLimit;
   private final int graphQLQueryDepthLimit;
@@ -527,6 +531,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.queryService = args.queryService;
     this.dataProductService = args.dataProductService;
     this.formService = args.formService;
+    this.restrictedService = args.restrictedService;
 
     this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration);
     this.authenticationConfiguration = Objects.requireNonNull(args.authenticationConfiguration);
@@ -576,6 +581,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.entityTypeType = new EntityTypeType(entityClient);
     this.formType = new FormType(entityClient);
     this.incidentType = new IncidentType(entityClient);
+    this.restrictedType = new RestrictedType(entityClient, restrictedService);
 
     this.graphQLQueryComplexityLimit = args.graphQLQueryComplexityLimit;
     this.graphQLQueryDepthLimit = args.graphQLQueryDepthLimit;
@@ -619,7 +625,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
             dataTypeType,
             entityTypeType,
             formType,
-            incidentType);
+            incidentType,
+            restrictedType);
     this.loadableTypes = new ArrayList<>(entityTypes);
     // Extend loadable types with types from the plugins
     // This allows us to offer search and browse capabilities out of the box for those types
@@ -709,6 +716,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
     configureStructuredPropertyResolvers(builder);
     configureFormResolvers(builder);
     configureIncidentResolvers(builder);
+    configureRestrictedResolvers(builder);
   }
 
   private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
@@ -1456,7 +1464,12 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
                     .dataFetcher(
                         "relationships", new EntityRelationshipsResultResolver(graphClient))
                     .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.datasetType))
-                    .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                    .dataFetcher(
+                        "lineage",
+                        new EntityLineageResultResolver(
+                            siblingGraphService,
+                            restrictedService,
+                            this.authorizationConfiguration))
                     .dataFetcher(
                         "platform",
                         new LoadableTypeResolver<>(
@@ -1823,7 +1836,10 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
             typeWiring
                 .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
                 .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.dashboardType))
-                .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                .dataFetcher(
+                    "lineage",
+                    new EntityLineageResultResolver(
+                        siblingGraphService, restrictedService, this.authorizationConfiguration))
                 .dataFetcher(
                     "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
                 .dataFetcher(
@@ -1950,7 +1966,10 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.chartType))
                 .dataFetcher(
                     "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
-                .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                .dataFetcher(
+                    "lineage",
+                    new EntityLineageResultResolver(
+                        siblingGraphService, restrictedService, this.authorizationConfiguration))
                 .dataFetcher(
                     "platform",
                     new LoadableTypeResolver<>(
@@ -2072,7 +2091,12 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) {
                     .dataFetcher(
                         "relationships", new EntityRelationshipsResultResolver(graphClient))
                     .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.dataJobType))
-                    .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                    .dataFetcher(
+                        "lineage",
+                        new EntityLineageResultResolver(
+                            siblingGraphService,
+                            restrictedService,
+                            this.authorizationConfiguration))
                     .dataFetcher(
                         "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
                     .dataFetcher(
@@ -2144,7 +2168,10 @@ private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) {
             typeWiring
                 .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
                 .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.dataFlowType))
-                .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                .dataFetcher(
+                    "lineage",
+                    new EntityLineageResultResolver(
+                        siblingGraphService, restrictedService, this.authorizationConfiguration))
                 .dataFetcher(
                     "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
                 .dataFetcher(
@@ -2188,7 +2215,12 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde
                         "browsePaths", new EntityBrowsePathsResolver(this.mlFeatureTableType))
                     .dataFetcher(
                         "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
-                    .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                    .dataFetcher(
+                        "lineage",
+                        new EntityLineageResultResolver(
+                            siblingGraphService,
+                            restrictedService,
+                            this.authorizationConfiguration))
                     .dataFetcher("exists", new EntityExistsResolver(entityService))
                     .dataFetcher(
                         "platform",
@@ -2271,7 +2303,12 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde
                     .dataFetcher(
                         "relationships", new EntityRelationshipsResultResolver(graphClient))
                     .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.mlModelType))
-                    .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                    .dataFetcher(
+                        "lineage",
+                        new EntityLineageResultResolver(
+                            siblingGraphService,
+                            restrictedService,
+                            this.authorizationConfiguration))
                     .dataFetcher("exists", new EntityExistsResolver(entityService))
                     .dataFetcher(
                         "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
@@ -2316,7 +2353,12 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde
                         "browsePaths", new EntityBrowsePathsResolver(this.mlModelGroupType))
                     .dataFetcher(
                         "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
-                    .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                    .dataFetcher(
+                        "lineage",
+                        new EntityLineageResultResolver(
+                            siblingGraphService,
+                            restrictedService,
+                            this.authorizationConfiguration))
                     .dataFetcher(
                         "platform",
                         new LoadableTypeResolver<>(
@@ -2339,7 +2381,12 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde
                 typeWiring
                     .dataFetcher(
                         "relationships", new EntityRelationshipsResultResolver(graphClient))
-                    .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                    .dataFetcher(
+                        "lineage",
+                        new EntityLineageResultResolver(
+                            siblingGraphService,
+                            restrictedService,
+                            this.authorizationConfiguration))
                     .dataFetcher(
                         "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
                     .dataFetcher("exists", new EntityExistsResolver(entityService))
@@ -2359,7 +2406,12 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde
                 typeWiring
                     .dataFetcher(
                         "relationships", new EntityRelationshipsResultResolver(graphClient))
-                    .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                    .dataFetcher(
+                        "lineage",
+                        new EntityLineageResultResolver(
+                            siblingGraphService,
+                            restrictedService,
+                            this.authorizationConfiguration))
                     .dataFetcher(
                         "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
                     .dataFetcher("exists", new EntityExistsResolver(entityService))
@@ -2656,7 +2708,10 @@ private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder b
         typeWiring ->
             typeWiring
                 .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
-                .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService))
+                .dataFetcher(
+                    "lineage",
+                    new EntityLineageResultResolver(
+                        siblingGraphService, restrictedService, this.authorizationConfiguration))
                 .dataFetcher(
                     "state",
                     new TimeSeriesAspectResolver(
@@ -2759,4 +2814,16 @@ private void configureIncidentResolvers(final RuntimeWiring.Builder builder) {
               typeWiring.dataFetcher("incidents", new EntityIncidentsResolver(entityClient)));
     }
   }
+
+  private void configureRestrictedResolvers(final RuntimeWiring.Builder builder) {
+    builder.type(
+        "Restricted",
+        typeWiring ->
+            typeWiring
+                .dataFetcher(
+                    "lineage",
+                    new EntityLineageResultResolver(
+                        siblingGraphService, restrictedService, this.authorizationConfiguration))
+                .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)));
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java

@@ -29,6 +29,7 @@
 import com.linkedin.metadata.service.LineageService;
 import com.linkedin.metadata.service.OwnershipTypeService;
 import com.linkedin.metadata.service.QueryService;
+import com.linkedin.metadata.service.RestrictedService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
 import com.linkedin.metadata.timeline.TimelineService;
@@ -75,6 +76,7 @@ public class GmsGraphQLEngineArgs {
   FeatureFlags featureFlags;
   DataProductService dataProductService;
   FormService formService;
+  RestrictedService restrictedService;
   int graphQLQueryComplexityLimit;
   int graphQLQueryDepthLimit;