Merge branch 'master' into feat_custom_pip_package_name

.github/actions/ci-optimization/action.yml

@@ -55,10 +55,11 @@ runs:
             - "metadata-ingestion/**"
             - "metadata-models/**"
             - "smoke-test/**"
-            - "docker/datahub-ingestion-**"
+            - "docker/datahub-ingestion**"
           docker:
             - "docker/**"
           backend:
+            - ".github/**"
             - "metadata-models/**"
             - "datahub-upgrade/**"
             - "entity-registry/**"

.github/actions/docker-custom-build-and-push/action.yml

@@ -20,7 +20,7 @@ inputs:
     required: false
 
   images:
-    # e.g. linkedin/datahub-gms
+    # e.g. acryldata/datahub-gms
     description: "List of Docker images to use as base name for tags"
     required: true
   build-args:

.github/workflows/airflow-plugin.yml

@@ -40,16 +40,16 @@ jobs:
             extra_pip_requirements: "apache-airflow~=2.2.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.4.0 pluggy==1.0.0 "pendulum<3.0" "Flask-Session<0.6.0"'
-            extra_pip_extras: plugin-v2
+            extra_pip_requirements: "apache-airflow==2.4.3"
+            extra_pip_extras: plugin-v2,test-airflow24
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.6.0 "pendulum<3.0" "Flask-Session<0.6.0"'
+            extra_pip_requirements: 'apache-airflow==2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.7.0 pydantic==2.4.2 "Flask-Session<0.6.0"'
+            extra_pip_requirements: 'apache-airflow==2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow>=2.8.0 pydantic>=2.4.2 "Flask-Session<0.6.0"'
+            extra_pip_requirements: 'apache-airflow==2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
@@ -67,7 +67,7 @@ jobs:
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
-        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze

README.md

@@ -27,7 +27,7 @@ HOSTED_DOCS_ONLY-->
 [![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest)
 [![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
 [![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
-[![Docker Pulls](https://img.shields.io/docker/pulls/linkedin/datahub-gms.svg)](https://hub.docker.com/r/linkedin/datahub-gms)
+[![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
 [![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://slack.datahubproject.io)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
 [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)

build.gradle

@@ -55,7 +55,7 @@ buildscript {
   ext.openLineageVersion = '1.5.0'
   ext.logbackClassicJava8 = '1.2.12'
 
-  ext.docker_registry = 'linkedin'
+  ext.docker_registry = 'acryldata'
 
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
@@ -95,6 +95,7 @@ project.ext.spec = [
             'restliDocgen' : 'com.linkedin.pegasus:restli-docgen:' + pegasusVersion,
             'restliServer' : 'com.linkedin.pegasus:restli-server:' + pegasusVersion,
             'restliSpringBridge': 'com.linkedin.pegasus:restli-spring-bridge:' + pegasusVersion,
+            'restliTestUtils' : 'com.linkedin.pegasus:restli-client-testutils:' + pegasusVersion,
         ]
     ]
 ]
@@ -246,6 +247,7 @@ project.ext.externalDependency = [
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
+    'swaggerCore': 'io.swagger.core.v3:swagger-core:2.2.7',
     'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18',
     'testng': 'org.testng:testng:7.8.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
@@ -264,7 +266,7 @@ project.ext.externalDependency = [
     'jline':'jline:jline:1.4.1',
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0',
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
-    'classGraph': 'io.github.classgraph:classgraph:4.8.165',
+    'classGraph': 'io.github.classgraph:classgraph:4.8.168',
 ]
 
 allprojects {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/EntityAssertionsResolver.java

@@ -4,6 +4,7 @@
 import com.linkedin.common.EntityRelationship;
 import com.linkedin.common.EntityRelationships;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.Assertion;
 import com.linkedin.datahub.graphql.generated.Entity;
@@ -25,8 +26,10 @@
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
 
 /** GraphQL Resolver used for fetching the list of Assertions associated with an Entity. */
+@Slf4j
 public class EntityAssertionsResolver
     implements DataFetcher<CompletableFuture<EntityAssertionsResult>> {
 
@@ -49,6 +52,8 @@ public CompletableFuture<EntityAssertionsResult> get(DataFetchingEnvironment env
           final String entityUrn = ((Entity) environment.getSource()).getUrn();
           final Integer start = environment.getArgumentOrDefault("start", 0);
           final Integer count = environment.getArgumentOrDefault("count", 200);
+          final Boolean includeSoftDeleted =
+              environment.getArgumentOrDefault("includeSoftDeleted", false);
 
           try {
             // Step 1: Fetch set of assertions associated with the target entity from the Graph
@@ -84,6 +89,7 @@ public CompletableFuture<EntityAssertionsResult> get(DataFetchingEnvironment env
                 gmsResults.stream()
                     .filter(Objects::nonNull)
                     .map(r -> AssertionMapper.map(context, r))
+                    .filter(assertion -> assertionExists(assertion, includeSoftDeleted, context))
                     .collect(Collectors.toList());
 
             // Step 4: Package and return result
@@ -98,4 +104,17 @@ public CompletableFuture<EntityAssertionsResult> get(DataFetchingEnvironment env
           }
         });
   }
+
+  private boolean assertionExists(
+      Assertion assertion, Boolean includeSoftDeleted, QueryContext context) {
+    try {
+      return _entityClient.exists(
+          UrnUtils.getUrn(assertion.getUrn()), includeSoftDeleted, context.getAuthentication());
+    } catch (RemoteInvocationException e) {
+      log.error(
+          String.format("Unable to check if assertion %s exists, ignoring it", assertion.getUrn()),
+          e);
+      return false;
+    }
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java

@@ -77,10 +77,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
                   .getFilters()
                   .forEach(
                       filter -> {
-                        criteria.add(
-                            new Criterion()
-                                .setField(filter.getField())
-                                .setValue(filter.getValue()));
+                        criteria.add(criterionFromFilter(filter, true));
                       });
             }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java

@@ -1,8 +1,10 @@
 package com.linkedin.datahub.graphql.resolvers.load;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
+import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*;
 
 import com.datahub.authorization.AuthorizationConfiguration;
+import com.linkedin.common.UrnArrayArray;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.SetMode;
@@ -156,6 +158,11 @@ private LineageRelationship mapEntityRelationship(
       result.setUpdatedActor(UrnToEntityMapper.map(context, updatedActor));
     }
     result.setIsManual(lineageRelationship.hasIsManual() && lineageRelationship.isIsManual());
+    if (lineageRelationship.getPaths() != null) {
+      UrnArrayArray paths = lineageRelationship.getPaths();
+      result.setPaths(
+          paths.stream().map(path -> mapPath(context, path)).collect(Collectors.toList()));
+    }
 
     return result;
   }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java

@@ -90,7 +90,8 @@ private SearchUtils() {}
           EntityType.CORP_USER,
           EntityType.CORP_GROUP,
           EntityType.NOTEBOOK,
-          EntityType.DATA_PRODUCT);
+          EntityType.DATA_PRODUCT,
+          EntityType.DOMAIN);
 
   /** Entities that are part of browse by default */
   public static final List<EntityType> BROWSE_ENTITY_TYPES =

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java

@@ -2,6 +2,7 @@
 
 import com.linkedin.assertion.AssertionInfo;
 import com.linkedin.common.DataPlatformInstance;
+import com.linkedin.common.Status;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.DataMap;
 import com.linkedin.datahub.graphql.QueryContext;
@@ -56,6 +57,18 @@ public static Assertion map(@Nullable QueryContext context, final EntityResponse
       result.setPlatform(unknownPlatform);
     }
 
+    final EnvelopedAspect envelopedStatus = aspects.get(Constants.STATUS_ASPECT_NAME);
+    if (envelopedStatus != null) {
+      result.setStatus(mapStatus(new Status(envelopedStatus.getValue().data())));
+    }
+
+    return result;
+  }
+
+  private static com.linkedin.datahub.graphql.generated.Status mapStatus(Status status) {
+    final com.linkedin.datahub.graphql.generated.Status result =
+        new com.linkedin.datahub.graphql.generated.Status();
+    result.setRemoved(status.isRemoved());
     return result;
   }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java

@@ -3,9 +3,11 @@
 import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*;
 import static com.linkedin.metadata.utils.SearchUtil.*;
 
+import com.linkedin.common.UrnArray;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.AggregationMetadata;
+import com.linkedin.datahub.graphql.generated.EntityPath;
 import com.linkedin.datahub.graphql.generated.FacetMetadata;
 import com.linkedin.datahub.graphql.generated.MatchedField;
 import com.linkedin.datahub.graphql.generated.SearchResult;
@@ -104,4 +106,11 @@ public static SearchSuggestion mapSearchSuggestion(
     return new SearchSuggestion(
         suggestion.getText(), suggestion.getScore(), Math.toIntExact(suggestion.getFrequency()));
   }
+
+  public static EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) {
+    EntityPath entityPath = new EntityPath();
+    entityPath.setPath(
+        path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList()));
+    return entityPath;
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java

@@ -3,11 +3,9 @@
 import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*;
 import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*;
 
-import com.linkedin.common.UrnArray;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.Entity;
-import com.linkedin.datahub.graphql.generated.EntityPath;
 import com.linkedin.datahub.graphql.generated.FreshnessStats;
 import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult;
 import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResults;
@@ -16,6 +14,7 @@
 import com.linkedin.metadata.search.LineageSearchEntity;
 import com.linkedin.metadata.search.LineageSearchResult;
 import com.linkedin.metadata.search.SearchResultMetadata;
+import java.util.ArrayList;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
@@ -69,14 +68,9 @@ private SearchAcrossLineageResult mapResult(
                 .map(p -> mapPath(context, p))
                 .collect(Collectors.toList()))
         .setDegree(searchEntity.getDegree())
-        .setDegrees(searchEntity.getDegrees().stream().collect(Collectors.toList()))
+        .setDegrees(new ArrayList<>(searchEntity.getDegrees()))
+        .setExplored(Boolean.TRUE.equals(searchEntity.isExplored()))
+        .setIgnoredAsHop(Boolean.TRUE.equals(searchEntity.isIgnoredAsHop()))
         .build();
   }
-
-  private EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) {
-    EntityPath entityPath = new EntityPath();
-    entityPath.setPath(
-        path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList()));
-    return entityPath;
-  }
 }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -1331,6 +1331,10 @@ type LineageRelationship {
     """
     isManual: Boolean
 
+    """
+    The paths traversed for this relationship
+    """
+    paths: [EntityPath]
 }
 
 """
@@ -1548,7 +1552,7 @@ type Dataset implements EntityWithRelationships & Entity & BrowsableEntity {
     """
     Assertions associated with the Dataset
     """
-    assertions(start: Int, count: Int): EntityAssertionsResult
+    assertions(start: Int, count: Int, includeSoftDeleted: Boolean): EntityAssertionsResult
 
     """
     Edges extending from this entity
@@ -7214,6 +7218,11 @@ type Assertion implements EntityWithRelationships & Entity {
     """
     lineage(input: LineageInput!): EntityLineageResult
 
+    """
+    Status metadata of the assertion
+    """
+    status: Status
+
     """
     Experimental API.
     For fetching extra aspects that do not have custom UI code yet

datahub-graphql-core/src/main/resources/search.graphql

@@ -644,7 +644,7 @@ type ScrollResults {
 }
 
 """
-Results returned by issueing a search across relationships query
+Results returned by issuing a search across relationships query
 """
 type SearchAcrossLineageResults {
   """
@@ -679,7 +679,7 @@ type SearchAcrossLineageResults {
 }
 
 """
-Results returned by issueing a search across relationships query using scroll API
+Results returned by issuing a search across relationships query using scroll API
 """
 type ScrollAcrossLineageResults {
   """
@@ -742,6 +742,16 @@ type SearchAcrossLineageResult {
   """
   degrees: [Int!]
 
+  """
+  Marks whether or not this entity was explored further for lineage
+  """
+  explored: Boolean!
+
+  """
+  Whether this relationship was ignored as a hop
+  """
+  ignoredAsHop: Boolean!
+
 }
 
 """

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/EntityAssertionsResolverTest.java

@@ -119,6 +119,11 @@ public void testGetSuccess() throws Exception {
                     .setUrn(assertionUrn)
                     .setAspects(new EnvelopedAspectMap(assertionAspects))));
 
+    Mockito.when(
+            mockClient.exists(
+                Mockito.any(Urn.class), Mockito.eq(false), Mockito.any(Authentication.class)))
+        .thenReturn(true);
+
     EntityAssertionsResolver resolver = new EntityAssertionsResolver(mockClient, graphClient);
 
     // Execute resolver
@@ -128,6 +133,8 @@ public void testGetSuccess() throws Exception {
 
     Mockito.when(mockEnv.getArgumentOrDefault(Mockito.eq("start"), Mockito.eq(0))).thenReturn(0);
     Mockito.when(mockEnv.getArgumentOrDefault(Mockito.eq("count"), Mockito.eq(200))).thenReturn(10);
+    Mockito.when(mockEnv.getArgumentOrDefault(Mockito.eq("includeSoftDeleted"), Mockito.eq(false)))
+        .thenReturn(false);
     Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
 
     Dataset parentEntity = new Dataset();
@@ -148,6 +155,9 @@ public void testGetSuccess() throws Exception {
     Mockito.verify(mockClient, Mockito.times(1))
         .batchGetV2(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
 
+    Mockito.verify(mockClient, Mockito.times(1))
+        .exists(Mockito.any(), Mockito.any(), Mockito.any());
+
     // Assert that GraphQL assertion run event matches expectations
     assertEquals(result.getStart(), 0);
     assertEquals(result.getCount(), 1);