Merge branch 'master' into add_connect_properties_from_env

.github/actions/ci-optimization/action.yml

@@ -17,6 +17,9 @@ outputs:
  ingestion-change:
  description: "Ingestion code has changed"
  value: ${{ steps.filter.outputs.ingestion == 'true' }}
+ ingestion-base-change:
+ description: "Ingestion base image docker image has changed"
+ value: ${{ steps.filter.outputs.ingestion-base == 'true' }}
  frontend-change:
  description: "Frontend code has changed"
  value: ${{ steps.filter.outputs.frontend == 'true' }}
@@ -56,6 +59,8 @@ runs:
  - "metadata-models/**"
  - "smoke-test/**"
  - "docker/datahub-ingestion**"
+ ingestion-base:
+ - "docker/datahub-ingestion-base/**"
  docker:
  - "docker/**"
  backend:

.gitignore

@@ -126,3 +126,6 @@ metadata-service/war/bin/
 metadata-utils/bin/
 test-models/bin/
 
+datahub-executor/
+datahub-integrations-service/
+metadata-ingestion-modules/acryl-cloud

build.gradle

@@ -32,29 +32,29 @@ buildscript {
 
  ext.junitJupiterVersion = '5.6.1'
  // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
- ext.pegasusVersion = '29.51.6'
+ ext.pegasusVersion = '29.57.0'
  ext.mavenVersion = '3.6.3'
- ext.springVersion = '6.1.4'
- ext.springBootVersion = '3.2.3'
- ext.springKafkaVersion = '3.1.2'
+ ext.springVersion = '6.1.5'
+ ext.springBootVersion = '3.2.6'
+ ext.springKafkaVersion = '3.1.6'
  ext.openTelemetryVersion = '1.18.0'
  ext.neo4jVersion = '5.14.0'
  ext.neo4jTestVersion = '5.14.0'
  ext.neo4jApocVersion = '5.14.0'
  ext.testContainersVersion = '1.17.4'
  ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
  ext.jacksonVersion = '2.15.3'
- ext.jettyVersion = '11.0.19'
- ext.playVersion = '2.8.21'
- ext.log4jVersion = '2.19.0'
+ ext.jettyVersion = '11.0.21'
+ ext.playVersion = '2.8.22'
+ ext.log4jVersion = '2.23.1'
  ext.slf4jVersion = '1.7.36'
  ext.logbackClassic = '1.4.14'
  ext.hadoop3Version = '3.3.5'
  ext.kafkaVersion = '5.5.15'
  ext.hazelcastVersion = '5.3.6'
  ext.ebeanVersion = '12.16.1'
  ext.googleJavaFormatVersion = '1.18.1'
- ext.openLineageVersion = '1.13.1'
+ ext.openLineageVersion = '1.16.0'
  ext.logbackClassicJava8 = '1.2.12'
 
  ext.docker_registry = 'acryldata'
@@ -149,7 +149,6 @@ project.ext.externalDependency = [
  'hazelcastTest':"com.hazelcast:hazelcast:$hazelcastVersion:tests",
  'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final',
  'httpClient': 'org.apache.httpcomponents.client5:httpclient5:5.3',
- 'httpAsyncClient': 'org.apache.httpcomponents:httpasyncclient:4.1.5',
  'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
  'jacksonJDK8': "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion",
  'jacksonDataPropertyFormat': "com.fasterxml.jackson.dataformat:jackson-dataformat-properties:$jacksonVersion",
@@ -250,6 +249,7 @@ project.ext.externalDependency = [
  'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
  'springKafka': "org.springframework.kafka:spring-kafka:$springKafkaVersion",
  'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
+ 'springRetry': "org.springframework.retry:spring-retry:2.0.6",
  'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
  'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
  'swaggerCore': 'io.swagger.core.v3:swagger-core:2.2.7',

datahub-frontend/app/auth/AuthModule.java

@@ -63,6 +63,7 @@ public class AuthModule extends AbstractModule {
  private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval";
  private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries";
  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize";
+ private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = "entityClient.restli.get.batchConcurrency";
  private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings";
 
  private final com.typesafe.config.Config _configs;
@@ -208,7 +209,8 @@ protected SystemEntityClient provideEntityClient(
  new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
  _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
  configurationProvider.getCache().getClient().getEntityClient(),
- Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)));
+ Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
+ Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
  }
 
  @Provides

datahub-frontend/app/controllers/Application.java

@@ -155,7 +155,7 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req
  .setBody(
  new InMemoryBodyWritable(
  ByteString.fromByteBuffer(request.body().asBytes().asByteBuffer()),
- "application/json"))
+ request.contentType().orElse("application/json")))
  .setRequestTimeout(Duration.ofSeconds(120))
  .execute()
  .thenApply(

datahub-frontend/conf/application.conf

@@ -38,8 +38,12 @@ jwt {
 play.server.provider = server.CustomAkkaHttpServerProvider
 play.http.server.akka.max-header-count = 64
 play.http.server.akka.max-header-count = ${?DATAHUB_AKKA_MAX_HEADER_COUNT}
-play.server.akka.max-header-size = 8k
+# max-header-size is reportedly no longer used
+play.server.akka.max-header-size = 32k
 play.server.akka.max-header-size = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH}
+# max header value length seems to impact the actual limit
+play.server.akka.max-header-value-length = 32k
+play.server.akka.max-header-value-length = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH}
 
 # Update AUTH_COOKIE_SAME_SITE and AUTH_COOKIE_SECURE in order to change how authentication cookies
 # are configured. If you wish cookies to be sent in first and third party contexts, set
@@ -289,5 +293,7 @@ entityClient.retryInterval = 2
 entityClient.retryInterval = ${?ENTITY_CLIENT_RETRY_INTERVAL}
 entityClient.numRetries = 3
 entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES}
-entityClient.restli.get.batchSize = 100
-entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE}
+entityClient.restli.get.batchSize = 50
+entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE}
+entityClient.restli.get.batchConcurrency = 2
+entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -1,5 +1,8 @@
 package com.linkedin.datahub.graphql;
 
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+
 /** Constants relating to GraphQL type system & execution. */
 public class Constants {
 
@@ -20,10 +23,19 @@ private Constants() {}
  public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql";
  public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
  public static final String FORMS_SCHEMA_FILE = "forms.graphql";
+ public static final String ASSERTIONS_SCHEMA_FILE = "assertions.graphql";
  public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql";
+ public static final String CONTRACTS_SCHEMA_FILE = "contract.graphql";
  public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql";
  public static final String BROWSE_PATH_DELIMITER = "/";
  public static final String BROWSE_PATH_V2_DELIMITER = "␟";
  public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";
  public static final String ENTITY_FILTER_NAME = "_entityType";
+
+ public static final Set<String> DEFAULT_PERSONA_URNS =
+ ImmutableSet.of(
+ "urn:li:dataHubPersona:technicalUser",
+ "urn:li:dataHubPersona:businessUser",
+ "urn:li:dataHubPersona:dataLeader",
+ "urn:li:dataHubPersona:dataSteward");
 }