Merge branch 'master' into feature-dremio-connector

.github/workflows/docker-unified.yml

@@ -990,7 +990,7 @@ jobs:
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
           ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
-          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
+          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions"
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
         run: |
           ./smoke-test/run-quickstart.sh
@@ -1030,6 +1030,7 @@ jobs:
           TEST_STRATEGY: ${{ matrix.test_strategy }}
         run: |
           echo "$DATAHUB_VERSION"
+          ./gradlew --stop
           ./smoke-test/smoke.sh
       - name: Disk Check
         run: df -h . && docker images
@@ -1043,8 +1044,9 @@ jobs:
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: docker logs
+          name: docker-logs-${{ matrix.test_strategy }}
           path: "docker_logs/*.log"
+          retention-days: 5
       - name: Upload screenshots
         uses: actions/upload-artifact@v3
         if: failure()

.github/workflows/pr-labeler.yml

@@ -61,18 +61,14 @@ jobs:
             contains(
               fromJson('[
                 "siladitya2",
-                "sgomezvillamor",
-                "ngamanda",
-                "HarveyLeo",
-                "frsann",
                 "bossenti",
-                "nikolakasev",
                 "PatrickfBraz",
                 "cuong-pham",
                 "sudhakarast",
                 "tkdrahn",
                 "rtekal",
-                "sgm44"
+                "mikeburke24",
+                "DSchmidtDev"
               ]'), 
               github.actor
             ) 

build.gradle

@@ -34,8 +34,8 @@ buildscript {
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
   ext.pegasusVersion = '29.57.0'
   ext.mavenVersion = '3.6.3'
-  ext.springVersion = '6.1.6'
-  ext.springBootVersion = '3.2.6'
+  ext.springVersion = '6.1.13'
+  ext.springBootVersion = '3.2.9'
   ext.springKafkaVersion = '3.1.6'
   ext.openTelemetryVersion = '1.18.0'
   ext.neo4jVersion = '5.14.0'
@@ -52,7 +52,7 @@ buildscript {
   ext.hadoop3Version = '3.3.6'
   ext.kafkaVersion = '5.5.15'
   ext.hazelcastVersion = '5.3.6'
-  ext.ebeanVersion = '12.16.1'
+  ext.ebeanVersion = '15.5.2'
   ext.googleJavaFormatVersion = '1.18.1'
   ext.openLineageVersion = '1.19.0'
   ext.logbackClassicJava8 = '1.2.12'
@@ -63,7 +63,7 @@ buildscript {
   buildscript.repositories.addAll(project.repositories)
   dependencies {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
-    classpath 'com.github.node-gradle:gradle-node-plugin:7.0.1'
+    classpath 'com.github.node-gradle:gradle-node-plugin:7.0.2'
     classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.2.0'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
@@ -104,8 +104,8 @@ project.ext.spec = [
 
 project.ext.externalDependency = [
     'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
-    'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2',
-    'antlr4': 'org.antlr:antlr4:4.7.2',
+    'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
+    'antlr4': 'org.antlr:antlr4:4.9.3',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.3',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
@@ -129,8 +129,10 @@ project.ext.externalDependency = [
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
     'ebean': 'io.ebean:ebean:' + ebeanVersion,
+    'ebeanTest': 'io.ebean:ebean-test:' + ebeanVersion,
     'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion,
     'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion,
+    'ebeanQueryBean': 'io.ebean:querybean-generator:' + ebeanVersion,
     'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion,
     'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
@@ -220,10 +222,10 @@ project.ext.externalDependency = [
     'playServer': "com.typesafe.play:play-server_2.12:$playVersion",
     'playTest': "com.typesafe.play:play-test_2.12:$playVersion",
     'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
-    'pac4j': 'org.pac4j:pac4j-oidc:4.5.7',
+    'pac4j': 'org.pac4j:pac4j-oidc:4.5.8',
     'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
     'postgresql': 'org.postgresql:postgresql:42.3.9',
-    'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
+    'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
     'reflections': 'org.reflections:reflections:0.9.9',
@@ -265,7 +267,7 @@ project.ext.externalDependency = [
     'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0',
     'typesafeConfig':'com.typesafe:config:1.4.1',
     'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
-    'zookeeper': 'org.apache.zookeeper:zookeeper:3.7.2',
+    'zookeeper': 'org.apache.zookeeper:zookeeper:3.8.4',
     'wire': 'com.squareup.wire:wire-compiler:3.7.1',
     'charle':  'com.charleskorn.kaml:kaml:0.53.0',
     'common': 'commons-io:commons-io:2.7',
@@ -359,6 +361,9 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
     exclude group: "org.slf4j", module: "slf4j-log4j12"
     exclude group: "org.slf4j", module: "slf4j-nop"
     exclude group: "org.slf4j", module: "slf4j-ext"
+
+    resolutionStrategy.force externalDependency.antlr4Runtime
+    resolutionStrategy.force externalDependency.antlr4
   }
 }
 

datahub-frontend/app/auth/sso/oidc/OidcConfigs.java

@@ -243,6 +243,9 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
             Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));
       }
 
+      grantType = Optional.ofNullable(getOptional(configs, OIDC_GRANT_TYPE, null));
+      acrValues = Optional.ofNullable(getOptional(configs, OIDC_ACR_VALUES, null));
+
       return this;
     }
 

datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java

@@ -18,7 +18,7 @@ public CustomOidcClient(final OidcConfiguration configuration) {
   protected void clientInit() {
     CommonHelper.assertNotNull("configuration", getConfiguration());
     getConfiguration().init();
-    defaultRedirectionActionBuilder(new OidcRedirectionActionBuilder(getConfiguration(), this));
+    defaultRedirectionActionBuilder(new CustomOidcRedirectionActionBuilder(getConfiguration(), this));
     defaultCredentialsExtractor(new OidcExtractor(getConfiguration(), this));
     defaultAuthenticator(new CustomOidcAuthenticator(this));
     defaultProfileCreator(new OidcProfileCreator<>(getConfiguration(), this));

datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java

@@ -0,0 +1,46 @@
+package auth.sso.oidc.custom;
+
+import java.util.Map;
+import java.util.Optional;
+import org.pac4j.core.context.WebContext;
+import org.pac4j.core.exception.http.RedirectionAction;
+import org.pac4j.core.exception.http.RedirectionActionHelper;
+import org.pac4j.oidc.client.OidcClient;
+import org.pac4j.oidc.config.OidcConfiguration;
+import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class CustomOidcRedirectionActionBuilder extends OidcRedirectionActionBuilder {
+
+  private static final Logger logger = LoggerFactory.getLogger(OidcRedirectionActionBuilder.class);
+  public CustomOidcRedirectionActionBuilder(OidcConfiguration configuration, OidcClient client) {
+    super(configuration, client);
+  }
+
+  @Override
+  public Optional<RedirectionAction> getRedirectionAction(WebContext context) {
+    Map<String, String> params = this.buildParams();
+    String computedCallbackUrl = this.client.computeFinalCallbackUrl(context);
+    params.put("redirect_uri", computedCallbackUrl);
+    this.addStateAndNonceParameters(context, params);
+    if (this.configuration.getMaxAge() != null) {
+      params.put("max_age", this.configuration.getMaxAge().toString());
+    }
+
+    String location = this.buildAuthenticationRequestUrl(params);
+
+    logger.debug("Custom parameters: {}", this.configuration.getCustomParams());
+
+    String acrValues = this.configuration.getCustomParam("acr_values");
+
+    if (acrValues != null && !location.contains("acr_values=")) {
+      location += (location.contains("?") ? "&" : "?") + "acr_values=" + acrValues;
+    }
+
+    logger.debug("Authentication request url: {}", location);
+    return Optional.of(RedirectionActionHelper.buildRedirectUrlAction(context, location));
+  }
+
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -293,6 +293,7 @@
 import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver;
 import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver;
 import com.linkedin.datahub.graphql.resolvers.structuredproperties.CreateStructuredPropertyResolver;
+import com.linkedin.datahub.graphql.resolvers.structuredproperties.DeleteStructuredPropertyResolver;
 import com.linkedin.datahub.graphql.resolvers.structuredproperties.RemoveStructuredPropertiesResolver;
 import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpdateStructuredPropertyResolver;
 import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpsertStructuredPropertiesResolver;
@@ -1034,6 +1035,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher("assertion", getResolver(assertionType))
                 .dataFetcher("form", getResolver(formType))
                 .dataFetcher("view", getResolver(dataHubViewType))
+                .dataFetcher("structuredProperty", getResolver(structuredPropertyType))
                 .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient))
                 .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver())
                 .dataFetcher("listUsers", new ListUsersResolver(this.entityClient))
@@ -1343,6 +1345,9 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
               .dataFetcher(
                   "updateStructuredProperty",
                   new UpdateStructuredPropertyResolver(this.entityClient))
+              .dataFetcher(
+                  "deleteStructuredProperty",
+                  new DeleteStructuredPropertyResolver(this.entityClient))
               .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient))
               .dataFetcher(
                   "updateIncidentStatus",
@@ -2116,6 +2121,9 @@ private void configureStructuredPropertyResolvers(final RuntimeWiring.Builder bu
                             .getAllowedTypes().stream()
                                 .map(entityTypeType.getKeyProvider())
                                 .collect(Collectors.toList()))));
+    builder.type(
+        "StructuredPropertyEntity",
+        typeWiring -> typeWiring.dataFetcher("exists", new EntityExistsResolver(entityService)));
   }
 
   /**

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java

@@ -78,7 +78,8 @@ public List<NamedLine> getTimeseriesChart(
       Optional<String> dimension, // Length 1 for now
       Map<String, List<String>> filters,
       Map<String, List<String>> mustNotFilters,
-      Optional<String> uniqueOn) {
+      Optional<String> uniqueOn,
+      String dateRangeField) {
 
     log.debug(
         String.format(
@@ -87,11 +88,11 @@ public List<NamedLine> getTimeseriesChart(
             + String.format("filters: %s, uniqueOn: %s", filters, uniqueOn));
 
     AggregationBuilder filteredAgg =
-        getFilteredAggregation(filters, mustNotFilters, Optional.of(dateRange));
+        getFilteredAggregation(filters, mustNotFilters, Optional.of(dateRange), dateRangeField);
 
     AggregationBuilder dateHistogram =
         AggregationBuilders.dateHistogram(DATE_HISTOGRAM)
-            .field("timestamp")
+            .field(dateRangeField)
             .calendarInterval(new DateHistogramInterval(granularity.name().toLowerCase()));
     uniqueOn.ifPresent(s -> dateHistogram.subAggregation(getUniqueQuery(s)));
 
@@ -128,6 +129,25 @@ public List<NamedLine> getTimeseriesChart(
     }
   }
 
+  public List<NamedLine> getTimeseriesChart(
+      String indexName,
+      DateRange dateRange,
+      DateInterval granularity,
+      Optional<String> dimension, // Length 1 for now
+      Map<String, List<String>> filters,
+      Map<String, List<String>> mustNotFilters,
+      Optional<String> uniqueOn) {
+    return getTimeseriesChart(
+        indexName,
+        dateRange,
+        granularity,
+        dimension,
+        filters,
+        mustNotFilters,
+        uniqueOn,
+        "timestamp");
+  }
+
   private int extractCount(MultiBucketsAggregation.Bucket bucket, boolean didUnique) {
     return didUnique
         ? (int) bucket.getAggregations().<Cardinality>get(UNIQUE).getValue()
@@ -323,20 +343,38 @@ private Filter executeAndExtract(SearchRequest searchRequest) {
     }
   }
 
+  // Make dateRangeField as customizable
   private AggregationBuilder getFilteredAggregation(
       Map<String, List<String>> mustFilters,
       Map<String, List<String>> mustNotFilters,
-      Optional<DateRange> dateRange) {
+      Optional<DateRange> dateRange,
+      String dateRangeField) {
     BoolQueryBuilder filteredQuery = QueryBuilders.boolQuery();
     mustFilters.forEach((key, values) -> filteredQuery.must(QueryBuilders.termsQuery(key, values)));
     mustNotFilters.forEach(
         (key, values) -> filteredQuery.mustNot(QueryBuilders.termsQuery(key, values)));
-    dateRange.ifPresent(range -> filteredQuery.must(dateRangeQuery(range)));
+    dateRange.ifPresent(range -> filteredQuery.must(dateRangeQuery(range, dateRangeField)));
     return AggregationBuilders.filter(FILTERED, filteredQuery);
   }
 
+  private AggregationBuilder getFilteredAggregation(
+      Map<String, List<String>> mustFilters,
+      Map<String, List<String>> mustNotFilters,
+      Optional<DateRange> dateRange) {
+    // Use timestamp as dateRangeField
+    return getFilteredAggregation(mustFilters, mustNotFilters, dateRange, "timestamp");
+  }
+
   private QueryBuilder dateRangeQuery(DateRange dateRange) {
-    return QueryBuilders.rangeQuery("timestamp").gte(dateRange.getStart()).lt(dateRange.getEnd());
+    // Use timestamp as dateRangeField
+    return dateRangeQuery(dateRange, "timestamp");
+  }
+
+  // Make dateRangeField as customizable
+  private QueryBuilder dateRangeQuery(DateRange dateRange, String dateRangeField) {
+    return QueryBuilders.rangeQuery(dateRangeField)
+        .gte(dateRange.getStart())
+        .lt(dateRange.getEnd());
   }
 
   private AggregationBuilder getUniqueQuery(String uniqueOn) {