From 1af41363bd4afb3bb3e8982701c12a60da0e38e8 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 21 Apr 2022 14:53:05 -0500 Subject: [PATCH 01/21] feat(versionedDataset): initial commit --- .../linkedin/datahub/graphql/Constants.java | 1 + .../datahub/graphql/GmsGraphQLEngine.java | 797 ++++++------------ .../resolvers/AuthenticatedResolver.java | 1 + .../resolvers/browse/BrowsePathsResolver.java | 4 +- .../resolvers/browse/BrowseResolver.java | 4 +- .../load/EntityTypeBatchResolver.java | 6 +- .../resolvers/load/EntityTypeResolver.java | 6 +- .../load/LoadableTypeBatchResolver.java | 6 +- .../resolvers/load/LoadableTypeResolver.java | 14 +- .../resolvers/load/OwnerTypeResolver.java | 6 +- .../AutoCompleteForMultipleResolver.java | 4 +- .../search/AutoCompleteResolver.java | 4 +- .../resolvers/search/AutocompleteUtils.java | 2 +- .../type/EntityInterfaceTypeResolver.java | 6 +- .../graphql/types/BrowsableEntityType.java | 2 +- .../datahub/graphql/types/EntityType.java | 2 +- .../datahub/graphql/types/LoadableType.java | 12 +- .../graphql/types/SearchableEntityType.java | 2 +- .../graphql/types/aspect/AspectType.java | 13 +- .../types/assertion/AssertionType.java | 2 +- .../graphql/types/chart/ChartType.java | 2 +- .../types/container/ContainerType.java | 2 +- .../types/corpgroup/CorpGroupType.java | 2 +- .../graphql/types/corpuser/CorpUserType.java | 2 +- .../types/dashboard/DashboardType.java | 2 +- .../graphql/types/dataflow/DataFlowType.java | 2 +- .../graphql/types/datajob/DataJobType.java | 2 +- .../types/dataplatform/DataPlatformType.java | 2 +- .../graphql/types/dataset/DatasetType.java | 19 +- .../graphql/types/domain/DomainType.java | 2 +- .../types/glossary/GlossaryTermType.java | 3 +- .../types/mlmodel/MLFeatureTableType.java | 3 +- .../graphql/types/mlmodel/MLFeatureType.java | 2 +- .../types/mlmodel/MLModelGroupType.java | 3 +- .../graphql/types/mlmodel/MLModelType.java | 2 +- .../types/mlmodel/MLPrimaryKeyType.java | 2 +- .../graphql/types/notebook/NotebookType.java | 3 +- .../datahub/graphql/types/tag/TagType.java | 3 +- .../graphql/types/usage/UsageType.java | 13 +- .../src/main/resources/entity.graphql | 2 +- .../models/registry/EntityRegistry.java | 2 +- li-utils/build.gradle | 2 +- .../java/com/linkedin/util/PairCoercer.java | 24 + .../main/pegasus/com/linkedin/common/Pair.pdl | 5 + .../metadata/entity/EntityService.java | 30 + .../linkedin/metadata/entity/EntityUtils.java | 25 + .../entity/ebean/EbeanEntityService.java | 29 +- .../timeline/data/ChangeTransaction.java | 1 + .../timeline/ebean/EbeanTimelineService.java | 99 ++- .../ebean/EbeanTimelineServiceTest.java | 2 +- .../timeline/TimelineServiceFactory.java | 7 +- ...n.entity.entitiesVersionedV2.restspec.json | 28 + .../com.linkedin.entity.aspects.snapshot.json | 10 +- ...com.linkedin.entity.entities.snapshot.json | 12 +- ...n.entity.entitiesVersionedV2.snapshot.json | 139 +++ .../com.linkedin.entity.runs.snapshot.json | 10 +- ...m.linkedin.platform.platform.snapshot.json | 12 +- .../linkedin/entity/client/EntityClient.java | 8 + .../entity/client/JavaEntityClient.java | 13 + .../entity/client/RestliEntityClient.java | 44 + .../resources/entity/EntityV2Resource.java | 11 +- .../entity/EntityVersionedV2Resource.java | 66 ++ .../resources/entity/ResourceUtils.java | 15 + .../resources/restli/RestliConstants.java | 2 + 64 files changed, 884 insertions(+), 679 deletions(-) create mode 100644 li-utils/src/main/java/com/linkedin/util/PairCoercer.java create mode 100644 li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl create mode 100644 metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesVersionedV2.restspec.json create mode 100644 metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json create mode 100644 metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java create mode 100644 metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java index aefe5d7c95691..724f08c067734 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java @@ -16,4 +16,5 @@ public class Constants { public static final String RECOMMENDATIONS_SCHEMA_FILE = "recommendation.graphql"; public static final String INGESTION_SCHEMA_FILE = "ingestion.graphql"; public static final String BROWSE_PATH_DELIMITER = "/"; + public static final String VERSION_STAMP_FIELD_NAME = "versionStamp"; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index a149f2c541b1a..d48ff270d64fe 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -11,7 +11,6 @@ import com.linkedin.datahub.graphql.analytics.service.AnalyticsService; import com.linkedin.datahub.graphql.generated.ActorFilter; import com.linkedin.datahub.graphql.generated.AggregationMetadata; -import com.linkedin.datahub.graphql.generated.Aspect; import com.linkedin.datahub.graphql.generated.Assertion; import com.linkedin.datahub.graphql.generated.AutoCompleteResultForEntity; import com.linkedin.datahub.graphql.generated.AutoCompleteResults; @@ -50,10 +49,8 @@ import com.linkedin.datahub.graphql.generated.RecommendationContent; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult; import com.linkedin.datahub.graphql.generated.SearchResult; -import com.linkedin.datahub.graphql.generated.UsageQueryResult; import com.linkedin.datahub.graphql.generated.UserUsageCounts; import com.linkedin.datahub.graphql.generated.VisualConfiguration; -import com.linkedin.datahub.graphql.resolvers.AuthenticatedResolver; import com.linkedin.datahub.graphql.resolvers.MeResolver; import com.linkedin.datahub.graphql.resolvers.assertion.AssertionRunEventResolver; import com.linkedin.datahub.graphql.resolvers.assertion.DeleteAssertionResolver; @@ -168,7 +165,10 @@ import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.version.GitVersion; import com.linkedin.usage.UsageClient; +import com.linkedin.util.Pair; import graphql.execution.DataFetcherResult; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; import graphql.schema.idl.RuntimeWiring; import java.io.IOException; import java.io.InputStream; @@ -188,14 +188,7 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; -import static com.linkedin.datahub.graphql.Constants.ANALYTICS_SCHEMA_FILE; -import static com.linkedin.datahub.graphql.Constants.APP_SCHEMA_FILE; -import static com.linkedin.datahub.graphql.Constants.AUTH_SCHEMA_FILE; -import static com.linkedin.datahub.graphql.Constants.GMS_SCHEMA_FILE; -import static com.linkedin.datahub.graphql.Constants.INGESTION_SCHEMA_FILE; -import static com.linkedin.datahub.graphql.Constants.RECOMMENDATIONS_SCHEMA_FILE; -import static com.linkedin.datahub.graphql.Constants.SEARCH_SCHEMA_FILE; -import static com.linkedin.datahub.graphql.Constants.URN_FIELD_NAME; +import static com.linkedin.datahub.graphql.Constants.*; import static com.linkedin.metadata.Constants.*; import static graphql.Scalars.GraphQLLong; @@ -250,47 +243,27 @@ public class GmsGraphQLEngine { /** * Configures the graph objects that can be fetched primary key. */ - public final List> entityTypes; + public final List> entityTypes; /** * Configures all graph objects */ - public final List> loadableTypes; + public final List> loadableTypes; /** * Configures the graph objects for owner */ - public final List> ownerTypes; + public final List> ownerTypes; /** * Configures the graph objects that can be searched. */ - public final List> searchableTypes; + public final List> searchableTypes; /** * Configures the graph objects that can be browsed. */ - public final List> browsableTypes; - - @Deprecated - public GmsGraphQLEngine() { - this( - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - false, - null); - } + public final List> browsableTypes; public GmsGraphQLEngine( final EntityClient entityClient, @@ -375,104 +348,20 @@ public GmsGraphQLEngine( this.loadableTypes = new ArrayList<>(entityTypes); this.ownerTypes = ImmutableList.of(corpUserType, corpGroupType); this.searchableTypes = loadableTypes.stream() - .filter(type -> (type instanceof SearchableEntityType)) - .map(type -> (SearchableEntityType) type) + .filter(type -> (type instanceof SearchableEntityType)) + .map(type -> (SearchableEntityType) type) .collect(Collectors.toList()); this.browsableTypes = loadableTypes.stream() - .filter(type -> (type instanceof BrowsableEntityType)) - .map(type -> (BrowsableEntityType) type) + .filter(type -> (type instanceof BrowsableEntityType)) + .map(type -> (BrowsableEntityType) type) .collect(Collectors.toList()); } - public static String entitySchema() { - String defaultSchemaString; - try { - InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(GMS_SCHEMA_FILE); - defaultSchemaString = IOUtils.toString(is, StandardCharsets.UTF_8); - is.close(); - } catch (IOException e) { - throw new RuntimeException("Failed to find GraphQL Schema with name " + GMS_SCHEMA_FILE, e); - } - return defaultSchemaString; - } - - public static String searchSchema() { - String defaultSchemaString; - try { - InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(SEARCH_SCHEMA_FILE); - defaultSchemaString = IOUtils.toString(is, StandardCharsets.UTF_8); - is.close(); - } catch (IOException e) { - throw new RuntimeException("Failed to find GraphQL Schema with name " + SEARCH_SCHEMA_FILE, e); - } - return defaultSchemaString; - } - - public static String appSchema() { - String defaultSchemaString; - try { - InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(APP_SCHEMA_FILE); - defaultSchemaString = IOUtils.toString(is, StandardCharsets.UTF_8); - is.close(); - } catch (IOException e) { - throw new RuntimeException("Failed to find GraphQL Schema with name " + APP_SCHEMA_FILE, e); - } - return defaultSchemaString; - } - - public static String authSchema() { - String defaultSchemaString; - try { - InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(AUTH_SCHEMA_FILE); - defaultSchemaString = IOUtils.toString(is, StandardCharsets.UTF_8); - is.close(); - } catch (IOException e) { - throw new RuntimeException("Failed to find GraphQL Schema with name " + AUTH_SCHEMA_FILE, e); - } - return defaultSchemaString; - } - - public static String analyticsSchema() { - String analyticsSchemaString; - try { - InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(ANALYTICS_SCHEMA_FILE); - analyticsSchemaString = IOUtils.toString(is, StandardCharsets.UTF_8); - is.close(); - } catch (IOException e) { - throw new RuntimeException("Failed to find GraphQL Schema with name " + ANALYTICS_SCHEMA_FILE, e); - } - return analyticsSchemaString; - } - - public static String recommendationsSchema() { - String recommendationsSchemaString; - try { - InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(RECOMMENDATIONS_SCHEMA_FILE); - recommendationsSchemaString = IOUtils.toString(is, StandardCharsets.UTF_8); - is.close(); - } catch (IOException e) { - throw new RuntimeException("Failed to find GraphQL Schema with name " + RECOMMENDATIONS_SCHEMA_FILE, e); - } - return recommendationsSchemaString; - } - - public static String ingestionSchema() { - String ingestionSchema; - try { - InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(INGESTION_SCHEMA_FILE); - ingestionSchema = IOUtils.toString(is, StandardCharsets.UTF_8); - is.close(); - } catch (IOException e) { - throw new RuntimeException("Failed to find GraphQL Schema with name " + INGESTION_SCHEMA_FILE, e); - } - return ingestionSchema; - } - /** * Returns a {@link Supplier} responsible for creating a new {@link DataLoader} from * a {@link LoadableType}. */ - public Map>> loaderSuppliers(final List> loadableTypes) { + public Map>> loaderSuppliers(final List> loadableTypes) { return loadableTypes .stream() .collect(Collectors.toMap( @@ -510,19 +399,31 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { public GraphQLEngine.Builder builder() { return GraphQLEngine.builder() - .addSchema(entitySchema()) - .addSchema(searchSchema()) - .addSchema(appSchema()) - .addSchema(authSchema()) - .addSchema(analyticsSchema()) - .addSchema(recommendationsSchema()) - .addSchema(ingestionSchema()) + .addSchema(fileBasedSchema(GMS_SCHEMA_FILE)) + .addSchema(fileBasedSchema(SEARCH_SCHEMA_FILE)) + .addSchema(fileBasedSchema(APP_SCHEMA_FILE)) + .addSchema(fileBasedSchema(AUTH_SCHEMA_FILE)) + .addSchema(fileBasedSchema(ANALYTICS_SCHEMA_FILE)) + .addSchema(fileBasedSchema(RECOMMENDATIONS_SCHEMA_FILE)) + .addSchema(fileBasedSchema(INGESTION_SCHEMA_FILE)) .addDataLoaders(loaderSuppliers(loadableTypes)) - .addDataLoader("Aspect", (context) -> createAspectLoader(context)) - .addDataLoader("UsageQueryResult", (context) -> createUsageLoader(context)) + .addDataLoader("Aspect", context -> createDataLoader(aspectType, context)) + .addDataLoader("UsageQueryResult", context -> createDataLoader(usageType, context)) .configureRuntimeWiring(this::configureRuntimeWiring); } + public static String fileBasedSchema(String fileName) { + String schema; + try { + InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName); + schema = IOUtils.toString(is, StandardCharsets.UTF_8); + is.close(); + } catch (IOException e) { + throw new RuntimeException("Failed to find GraphQL Schema with name " + fileName, e); + } + return schema; + } + private void configureAnalyticsResolvers(final RuntimeWiring.Builder builder) { final boolean isAnalyticsEnabled = analyticsService != null; builder.type("Query", typeWiring -> typeWiring.dataFetcher("isAnalyticsEnabled", new IsAnalyticsEnabledResolver(isAnalyticsEnabled))) @@ -564,123 +465,80 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { this.ingestionConfiguration, this.authorizationConfiguration, supportsImpactAnalysis, this.visualConfiguration)) - .dataFetcher("me", new AuthenticatedResolver<>( - new MeResolver(this.entityClient))) - .dataFetcher("search", new AuthenticatedResolver<>( - new SearchResolver(this.entityClient))) - .dataFetcher("searchAcrossEntities", - new SearchAcrossEntitiesResolver(this.entityClient)) - .dataFetcher("searchAcrossLineage", - new SearchAcrossLineageResolver(this.entityClient)) - .dataFetcher("autoComplete", new AuthenticatedResolver<>( - new AutoCompleteResolver(searchableTypes))) - .dataFetcher("autoCompleteForMultiple", new AuthenticatedResolver<>( - new AutoCompleteForMultipleResolver(searchableTypes))) - .dataFetcher("browse", new AuthenticatedResolver<>( - new BrowseResolver(browsableTypes))) - .dataFetcher("browsePaths", new AuthenticatedResolver<>( - new BrowsePathsResolver(browsableTypes))) - .dataFetcher("dataset", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(datasetType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("notebook", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(notebookType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("corpUser", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(corpUserType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("corpGroup", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(corpGroupType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("dashboard", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dashboardType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("chart", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(chartType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("tag", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(tagType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("dataFlow", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataFlowType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("dataJob", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataJobType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("glossaryTerm", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(glossaryTermType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("domain", - new LoadableTypeResolver<>(domainType, - (env) -> env.getArgument(URN_FIELD_NAME))) - .dataFetcher("dataPlatform", - new LoadableTypeResolver<>(dataPlatformType, - (env) -> env.getArgument(URN_FIELD_NAME))) - .dataFetcher("mlFeatureTable", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(mlFeatureTableType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("mlFeature", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(mlFeatureType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("mlPrimaryKey", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(mlPrimaryKeyType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("mlModel", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(mlModelType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("mlModelGroup", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(mlModelGroupType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("assertion", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(assertionType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("listPolicies", - new ListPoliciesResolver(this.entityClient)) + .dataFetcher("me", new MeResolver(this.entityClient)) + .dataFetcher("search", new SearchResolver(this.entityClient)) + .dataFetcher("searchAcrossEntities", new SearchAcrossEntitiesResolver(this.entityClient)) + .dataFetcher("searchAcrossLineage", new SearchAcrossLineageResolver(this.entityClient)) + .dataFetcher("autoComplete", new AutoCompleteResolver(searchableTypes)) + .dataFetcher("autoCompleteForMultiple", new AutoCompleteForMultipleResolver(searchableTypes)) + .dataFetcher("browse", new BrowseResolver(browsableTypes)) + .dataFetcher("browsePaths", new BrowsePathsResolver(browsableTypes)) + .dataFetcher("dataset", getResolver(datasetType, + (env) -> new Pair<>(env.getArgument(URN_FIELD_NAME), + env.getArgument(VERSION_STAMP_FIELD_NAME)))) + .dataFetcher("notebook", getResolver(notebookType)) + .dataFetcher("corpUser", getResolver(corpUserType)) + .dataFetcher("corpGroup", getResolver(corpGroupType)) + .dataFetcher("dashboard", getResolver(dashboardType)) + .dataFetcher("chart", getResolver(chartType)) + .dataFetcher("tag", getResolver(tagType)) + .dataFetcher("dataFlow", getResolver(dataFlowType)) + .dataFetcher("dataJob", getResolver(dataJobType)) + .dataFetcher("glossaryTerm", getResolver(glossaryTermType)) + .dataFetcher("domain", getResolver((domainType))) + .dataFetcher("dataPlatform", getResolver(dataPlatformType)) + .dataFetcher("mlFeatureTable", getResolver(mlFeatureTableType)) + .dataFetcher("mlFeature", getResolver(mlFeatureType)) + .dataFetcher("mlPrimaryKey", getResolver(mlPrimaryKeyType)) + .dataFetcher("mlModel", getResolver(mlModelType)) + .dataFetcher("mlModelGroup", getResolver(mlModelGroupType)) + .dataFetcher("assertion", getResolver(assertionType)) + .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient)) .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver()) - .dataFetcher("listUsers", - new ListUsersResolver(this.entityClient)) - .dataFetcher("listGroups", - new ListGroupsResolver(this.entityClient)) - .dataFetcher("listRecommendations", - new ListRecommendationsResolver(recommendationsService)) - .dataFetcher("getEntityCounts", - new EntityCountsResolver(this.entityClient)) - .dataFetcher("getAccessToken", - new GetAccessTokenResolver(tokenService)) - .dataFetcher("container", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(containerType, - (env) -> env.getArgument(URN_FIELD_NAME)))) - .dataFetcher("listDomains", - new ListDomainsResolver(this.entityClient)) - .dataFetcher("listSecrets", - new ListSecretsResolver(this.entityClient)) - .dataFetcher("getSecretValues", - new GetSecretValuesResolver(this.entityClient, this.secretService)) - .dataFetcher("listIngestionSources", - new ListIngestionSourcesResolver(this.entityClient)) - .dataFetcher("ingestionSource", - new GetIngestionSourceResolver(this.entityClient)) - .dataFetcher("executionRequest", - new GetIngestionExecutionRequestResolver(this.entityClient)) + .dataFetcher("listUsers", new ListUsersResolver(this.entityClient)) + .dataFetcher("listGroups", new ListGroupsResolver(this.entityClient)) + .dataFetcher("listRecommendations", new ListRecommendationsResolver(recommendationsService)) + .dataFetcher("getEntityCounts", new EntityCountsResolver(this.entityClient)) + .dataFetcher("getAccessToken", new GetAccessTokenResolver(tokenService)) + .dataFetcher("container", getResolver(containerType)) + .dataFetcher("listDomains", new ListDomainsResolver(this.entityClient)) + .dataFetcher("listSecrets", new ListSecretsResolver(this.entityClient)) + .dataFetcher("getSecretValues", new GetSecretValuesResolver(this.entityClient, this.secretService)) + .dataFetcher("listIngestionSources", new ListIngestionSourcesResolver(this.entityClient)) + .dataFetcher("ingestionSource", new GetIngestionSourceResolver(this.entityClient)) + .dataFetcher("executionRequest", new GetIngestionExecutionRequestResolver(this.entityClient)) ); } + private DataFetcher getResolver(LoadableType loadableType) { + return getResolver(loadableType, this::getUrnField); + } + + private DataFetcher getResolver(LoadableType loadableType, + Function keyProvider) { + return new LoadableTypeResolver<>(loadableType, keyProvider); + } + + private String getUrnField(DataFetchingEnvironment env) { + return env.getArgument(URN_FIELD_NAME); + } + private void configureMutationResolvers(final RuntimeWiring.Builder builder) { builder.type("Mutation", typeWiring -> typeWiring - .dataFetcher("updateDataset", new AuthenticatedResolver<>(new MutableTypeResolver<>(datasetType))) - .dataFetcher("updateTag", new AuthenticatedResolver<>(new MutableTypeResolver<>(tagType))) + .dataFetcher("updateDataset", new MutableTypeResolver<>(datasetType)) + .dataFetcher("updateTag", new MutableTypeResolver<>(tagType)) .dataFetcher("setTagColor", new SetTagColorResolver(entityClient, entityService)) - .dataFetcher("updateChart", new AuthenticatedResolver<>(new MutableTypeResolver<>(chartType))) - .dataFetcher("updateDashboard", new AuthenticatedResolver<>(new MutableTypeResolver<>(dashboardType))) - .dataFetcher("updateNotebook", new AuthenticatedResolver<>(new MutableTypeResolver<>(notebookType))) - .dataFetcher("updateDataJob", new AuthenticatedResolver<>(new MutableTypeResolver<>(dataJobType))) - .dataFetcher("updateDataFlow", new AuthenticatedResolver<>(new MutableTypeResolver<>(dataFlowType))) + .dataFetcher("updateChart", new MutableTypeResolver<>(chartType)) + .dataFetcher("updateDashboard", new MutableTypeResolver<>(dashboardType)) + .dataFetcher("updateNotebook", new MutableTypeResolver<>(notebookType)) + .dataFetcher("updateDataJob", new MutableTypeResolver<>(dataJobType)) + .dataFetcher("updateDataFlow", new MutableTypeResolver<>(dataFlowType)) .dataFetcher("updateCorpUserProperties", new MutableTypeResolver<>(corpUserType)) .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType)) - .dataFetcher("addTag", new AuthenticatedResolver<>(new AddTagResolver(entityService))) - .dataFetcher("removeTag", new AuthenticatedResolver<>(new RemoveTagResolver(entityService))) - .dataFetcher("addTerm", new AuthenticatedResolver<>(new AddTermResolver(entityService))) - .dataFetcher("removeTerm", new AuthenticatedResolver<>(new RemoveTermResolver(entityService))) + .dataFetcher("addTag", new AddTagResolver(entityService)) + .dataFetcher("removeTag", new RemoveTagResolver(entityService)) + .dataFetcher("addTerm", new AddTermResolver(entityService)) + .dataFetcher("removeTerm", new RemoveTermResolver(entityService)) .dataFetcher("createPolicy", new UpsertPolicyResolver(this.entityClient)) .dataFetcher("updatePolicy", new UpsertPolicyResolver(this.entityClient)) .dataFetcher("deletePolicy", new DeletePolicyResolver(this.entityClient)) @@ -713,18 +571,14 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder) { builder .type("SearchResult", typeWiring -> typeWiring - .dataFetcher("entity", new AuthenticatedResolver<>( - new EntityTypeResolver( + .dataFetcher("entity", new EntityTypeResolver( entityTypes.stream().collect(Collectors.toList()), (env) -> ((SearchResult) env.getSource()).getEntity())) ) - ) .type("SearchAcrossLineageResult", typeWiring -> typeWiring - .dataFetcher("entity", new AuthenticatedResolver<>( - new EntityTypeResolver( + .dataFetcher("entity", new EntityTypeResolver( entityTypes.stream().collect(Collectors.toList()), (env) -> ((SearchAcrossLineageResult) env.getSource()).getEntity())) - ) ) .type("AggregationMetadata", typeWiring -> typeWiring .dataFetcher("entity", new EntityTypeResolver( @@ -737,36 +591,27 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder (env) -> ((RecommendationContent) env.getSource()).getEntity())) ) .type("BrowseResults", typeWiring -> typeWiring - .dataFetcher("entities", new AuthenticatedResolver<>( - new EntityTypeBatchResolver( + .dataFetcher("entities", new EntityTypeBatchResolver( entityTypes.stream().collect(Collectors.toList()), (env) -> ((BrowseResults) env.getSource()).getEntities())) - ) ) .type("EntityRelationshipLegacy", typeWiring -> typeWiring - .dataFetcher("entity", new AuthenticatedResolver<>( - new EntityTypeResolver( + .dataFetcher("entity", new EntityTypeResolver( new ArrayList<>(entityTypes), (env) -> ((EntityRelationshipLegacy) env.getSource()).getEntity())) - ) ) .type("EntityRelationship", typeWiring -> typeWiring - .dataFetcher("entity", new AuthenticatedResolver<>( - new EntityTypeResolver( + .dataFetcher("entity", new EntityTypeResolver( new ArrayList<>(entityTypes), (env) -> ((EntityRelationship) env.getSource()).getEntity())) - ) ) .type("LineageRelationship", typeWiring -> typeWiring - .dataFetcher("entity", new AuthenticatedResolver<>( - new EntityTypeResolver( + .dataFetcher("entity", new EntityTypeResolver( new ArrayList<>(entityTypes), (env) -> ((LineageRelationship) env.getSource()).getEntity())) - ) ) .type("ListDomainsResult", typeWiring -> typeWiring - .dataFetcher("domains", - new LoadableTypeBatchResolver<>(domainType, + .dataFetcher("domains", new LoadableTypeBatchResolver<>(domainType, (env) -> ((ListDomainsResult) env.getSource()).getDomains().stream() .map(Domain::getUrn) .collect(Collectors.toList()))) @@ -778,17 +623,14 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder (env) -> ((AutoCompleteResults) env.getSource()).getEntities())) ) .type("AutoCompleteResultForEntity", typeWiring -> typeWiring - .dataFetcher("entities", - new EntityTypeBatchResolver( + .dataFetcher("entities", new EntityTypeBatchResolver( new ArrayList<>(entityTypes), (env) -> ((AutoCompleteResultForEntity) env.getSource()).getEntities())) ) .type("PolicyMatchCriterionValue", typeWiring -> typeWiring - .dataFetcher("entity", new AuthenticatedResolver<>( - new EntityTypeResolver( + .dataFetcher("entity", new EntityTypeResolver( new ArrayList<>(entityTypes), (env) -> ((PolicyMatchCriterionValue) env.getSource()).getEntity())) - ) ); } @@ -798,12 +640,8 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { builder .type("Dataset", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) .dataFetcher("domain", new LoadableTypeResolver<>( domainType, @@ -811,9 +649,8 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { final Dataset dataset = env.getSource(); return dataset.getDomain() != null ? dataset.getDomain().getUrn() : null; })) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, - (env) -> ((Dataset) env.getSource()).getPlatform().getUrn())) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, + (env) -> ((Dataset) env.getSource()).getPlatform().getUrn()) ) .dataFetcher("container", new LoadableTypeResolver<>(containerType, @@ -822,69 +659,52 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { return dataset.getContainer() != null ? dataset.getContainer().getUrn() : null; }) ) - .dataFetcher("datasetProfiles", new AuthenticatedResolver<>( - new TimeSeriesAspectResolver( + .dataFetcher("datasetProfiles", new TimeSeriesAspectResolver( this.entityClient, "dataset", "datasetProfile", DatasetProfileMapper::map ) - )) - .dataFetcher("operations", new AuthenticatedResolver<>( - new TimeSeriesAspectResolver( + ) + .dataFetcher("operations", new TimeSeriesAspectResolver( this.entityClient, "dataset", "operation", OperationMapper::map ) - )) - .dataFetcher("usageStats", new AuthenticatedResolver<>(new UsageTypeResolver())) - .dataFetcher("health", new DatasetHealthResolver(graphClient, timeseriesAspectService)) - .dataFetcher("schemaMetadata", new AuthenticatedResolver<>( - new AspectResolver()) ) + .dataFetcher("usageStats", new UsageTypeResolver()) + .dataFetcher("health", new DatasetHealthResolver(graphClient, timeseriesAspectService)) + .dataFetcher("schemaMetadata", new AspectResolver()) .dataFetcher("assertions", new EntityAssertionsResolver(entityClient, graphClient)) - .dataFetcher("aspects", new AuthenticatedResolver<>( - new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) - ) - .dataFetcher("subTypes", new AuthenticatedResolver(new SubTypesResolver( + .dataFetcher("aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) + .dataFetcher("subTypes", new SubTypesResolver( this.entityClient, "dataset", - "subTypes"))) - .dataFetcher("runs", new EntityRunsResolver(entityClient)) - ) + "subTypes")) + .dataFetcher("runs", new EntityRunsResolver(entityClient))) .type("Owner", typeWiring -> typeWiring - .dataFetcher("owner", new AuthenticatedResolver<>( - new OwnerTypeResolver<>(ownerTypes, - (env) -> ((Owner) env.getSource()).getOwner())) - ) + .dataFetcher("owner", new OwnerTypeResolver<>(ownerTypes, + (env) -> ((Owner) env.getSource()).getOwner())) ) .type("UserUsageCounts", typeWiring -> typeWiring - .dataFetcher("user", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(corpUserType, - (env) -> ((UserUsageCounts) env.getSource()).getUser().getUrn())) - ) + .dataFetcher("user", new LoadableTypeResolver<>(corpUserType, + (env) -> ((UserUsageCounts) env.getSource()).getUser().getUrn())) ) .type("ForeignKeyConstraint", typeWiring -> typeWiring - .dataFetcher("foreignDataset", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(datasetType, - (env) -> ((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn())) - ) + .dataFetcher("foreignDataset", new LoadableTypeResolver<>(datasetType, + (env) -> new Pair<>(((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn(), null))) ) .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring - .dataFetcher("author", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(corpUserType, - (env) -> ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())) - ) + .dataFetcher("author", new LoadableTypeResolver<>(corpUserType, + (env) -> ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())) ); } private void configureGlossaryTermResolvers(final RuntimeWiring.Builder builder) { builder.type("GlossaryTerm", typeWiring -> typeWiring - .dataFetcher("schemaMetadata", new AuthenticatedResolver<>( - new AspectResolver()) - ) + .dataFetcher("schemaMetadata", new AspectResolver()) ); } @@ -893,15 +713,12 @@ private void configureGlossaryTermResolvers(final RuntimeWiring.Builder builder) */ private void configureCorpUserResolvers(final RuntimeWiring.Builder builder) { builder.type("CorpUser", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) + .dataFetcher("relationships", + new EntityRelationshipsResultResolver(graphClient)) ); builder.type("CorpUserInfo", typeWiring -> typeWiring - .dataFetcher("manager", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(corpUserType, - (env) -> ((CorpUserInfo) env.getSource()).getManager().getUrn())) - ) + .dataFetcher("manager", new LoadableTypeResolver<>(corpUserType, + (env) -> ((CorpUserInfo) env.getSource()).getManager().getUrn())) ); } @@ -910,37 +727,28 @@ private void configureCorpUserResolvers(final RuntimeWiring.Builder builder) { */ private void configureCorpGroupResolvers(final RuntimeWiring.Builder builder) { builder.type("CorpGroup", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - ); + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))); builder.type("CorpGroupInfo", typeWiring -> typeWiring - .dataFetcher("admins", new AuthenticatedResolver<>( + .dataFetcher("admins", new LoadableTypeBatchResolver<>(corpUserType, (env) -> ((CorpGroupInfo) env.getSource()).getAdmins().stream() .map(CorpUser::getUrn) .collect(Collectors.toList()))) - ) - .dataFetcher("members", new AuthenticatedResolver<>( + .dataFetcher("members", new LoadableTypeBatchResolver<>(corpUserType, (env) -> ((CorpGroupInfo) env.getSource()).getMembers().stream() .map(CorpUser::getUrn) .collect(Collectors.toList()))) - ) ); } private void configureTagAssociationResolver(final RuntimeWiring.Builder builder) { builder.type("Tag", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - ); + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))); builder.type("TagAssociation", typeWiring -> typeWiring - .dataFetcher("tag", new AuthenticatedResolver<>( + .dataFetcher("tag", new LoadableTypeResolver<>(tagType, (env) -> ((com.linkedin.datahub.graphql.generated.TagAssociation) env.getSource()).getTag().getUrn())) - ) ); } @@ -949,18 +757,12 @@ private void configureTagAssociationResolver(final RuntimeWiring.Builder builder */ private void configureNotebookResolvers(final RuntimeWiring.Builder builder) { builder.type("Notebook", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, - (env) -> ((Notebook) env.getSource()).getPlatform().getUrn())) - ) - .dataFetcher("domain", - new LoadableTypeResolver<>(domainType, - (env) -> ((Notebook) env.getSource()).getDomain().getUrn()) - ) - ); + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, + (env) -> ((Notebook) env.getSource()).getPlatform().getUrn())) + .dataFetcher("domain", new LoadableTypeResolver<>(domainType, + (env) -> ((Notebook) env.getSource()).getDomain().getUrn()) + )); } /** @@ -968,18 +770,11 @@ private void configureNotebookResolvers(final RuntimeWiring.Builder builder) { */ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { builder.type("Dashboard", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((Dashboard) env.getSource()).getPlatform().getUrn())) - ) - .dataFetcher("domain", - new LoadableTypeResolver<>( + .dataFetcher("domain", new LoadableTypeResolver<>( domainType, (env) -> { final Dashboard dashboard = env.getSource(); @@ -987,8 +782,7 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { } ) ) - .dataFetcher("container", - new LoadableTypeResolver<>(containerType, + .dataFetcher("container", new LoadableTypeResolver<>(containerType, (env) -> { final Dashboard dashboard = env.getSource(); return dashboard.getContainer() != null ? dashboard.getContainer().getUrn() : null; @@ -996,12 +790,10 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { ) ); builder.type("DashboardInfo", typeWiring -> typeWiring - .dataFetcher("charts", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(chartType, + .dataFetcher("charts", new LoadableTypeBatchResolver<>(chartType, (env) -> ((DashboardInfo) env.getSource()).getCharts().stream() .map(Chart::getUrn) .collect(Collectors.toList()))) - ) ); } @@ -1010,40 +802,30 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { */ private void configureChartResolvers(final RuntimeWiring.Builder builder) { builder.type("Chart", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, - (env) -> ((Chart) env.getSource()).getPlatform().getUrn())) - ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final Chart chart = env.getSource(); - return chart.getDomain() != null ? chart.getDomain().getUrn() : null; - }) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, + (env) -> ((Chart) env.getSource()).getPlatform().getUrn())) + .dataFetcher("domain", new LoadableTypeResolver<>( + domainType, + (env) -> { + final Chart chart = env.getSource(); + return chart.getDomain() != null ? chart.getDomain().getUrn() : null; + }) ) - .dataFetcher("container", - new LoadableTypeResolver<>( - containerType, - (env) -> { - final Chart chart = env.getSource(); - return chart.getContainer() != null ? chart.getContainer().getUrn() : null; - }) + .dataFetcher("container", new LoadableTypeResolver<>( + containerType, + (env) -> { + final Chart chart = env.getSource(); + return chart.getContainer() != null ? chart.getContainer().getUrn() : null; + }) ) ); builder.type("ChartInfo", typeWiring -> typeWiring - .dataFetcher("inputs", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(datasetType, - (env) -> ((ChartInfo) env.getSource()).getInputs().stream() - .map(Dataset::getUrn) - .collect(Collectors.toList()))) - ) + .dataFetcher("inputs", new LoadableTypeBatchResolver<>(datasetType, + (env) -> ((ChartInfo) env.getSource()).getInputs().stream() + .map(Dataset::getUrn) + .collect(Collectors.toList()))) ); } @@ -1055,19 +837,19 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) { .type("Entity", typeWiring -> typeWiring .typeResolver(new EntityInterfaceTypeResolver(loadableTypes.stream() .filter(graphType -> graphType instanceof EntityType) - .map(graphType -> (EntityType) graphType) + .map(graphType -> (EntityType) graphType) .collect(Collectors.toList()) ))) .type("EntityWithRelationships", typeWiring -> typeWiring .typeResolver(new EntityInterfaceTypeResolver(loadableTypes.stream() .filter(graphType -> graphType instanceof EntityType) - .map(graphType -> (EntityType) graphType) + .map(graphType -> (EntityType) graphType) .collect(Collectors.toList()) ))) .type("OwnerType", typeWiring -> typeWiring .typeResolver(new EntityInterfaceTypeResolver(ownerTypes.stream() .filter(graphType -> graphType instanceof EntityType) - .map(graphType -> (EntityType) graphType) + .map(graphType -> (EntityType) graphType) .collect(Collectors.toList()) ))) .type("PlatformSchema", typeWiring -> typeWiring @@ -1077,7 +859,8 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) { .typeResolver(new HyperParameterValueTypeResolver()) ) .type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver())) - .type("TimeSeriesAspect", typeWiring -> typeWiring.typeResolver(new TimeSeriesAspectInterfaceTypeResolver())) + .type("TimeSeriesAspect", typeWiring -> typeWiring + .typeResolver(new TimeSeriesAspectInterfaceTypeResolver())) .type("ResultsType", typeWiring -> typeWiring .typeResolver(new ResultsTypeResolver())); } @@ -1095,45 +878,32 @@ private void configureTypeExtensions(final RuntimeWiring.Builder builder) { private void configureDataJobResolvers(final RuntimeWiring.Builder builder) { builder .type("DataJob", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("dataFlow", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataFlowType, - (env) -> ((DataJob) env.getSource()).getDataFlow().getUrn())) - ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final DataJob dataJob = env.getSource(); - return dataJob.getDomain() != null ? dataJob.getDomain().getUrn() : null; - }) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("dataFlow", new LoadableTypeResolver<>(dataFlowType, + (env) -> ((DataJob) env.getSource()).getDataFlow().getUrn())) + .dataFetcher("domain", new LoadableTypeResolver<>( + domainType, + (env) -> { + final DataJob dataJob = env.getSource(); + return dataJob.getDomain() != null ? dataJob.getDomain().getUrn() : null; + }) ) .dataFetcher("runs", new DataJobRunsResolver(entityClient)) ) .type("DataJobInputOutput", typeWiring -> typeWiring - .dataFetcher("inputDatasets", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(datasetType, - (env) -> ((DataJobInputOutput) env.getSource()).getInputDatasets().stream() - .map(Dataset::getUrn) - .collect(Collectors.toList()))) - ) - .dataFetcher("outputDatasets", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(datasetType, - (env) -> ((DataJobInputOutput) env.getSource()).getOutputDatasets().stream() - .map(Dataset::getUrn) - .collect(Collectors.toList()))) - ) - .dataFetcher("inputDatajobs", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(dataJobType, - (env) -> ((DataJobInputOutput) env.getSource()).getInputDatajobs().stream() - .map(DataJob::getUrn) - .collect(Collectors.toList()))) - ) + .dataFetcher("inputDatasets", new LoadableTypeBatchResolver<>(datasetType, + (env) -> ((DataJobInputOutput) env.getSource()).getInputDatasets().stream() + .map(Dataset::getUrn) + .collect(Collectors.toList()))) + .dataFetcher("outputDatasets", new LoadableTypeBatchResolver<>(datasetType, + (env) -> ((DataJobInputOutput) env.getSource()).getOutputDatasets().stream() + .map(Dataset::getUrn) + .collect(Collectors.toList()))) + .dataFetcher("inputDatajobs", new LoadableTypeBatchResolver<>(dataJobType, + (env) -> ((DataJobInputOutput) env.getSource()).getInputDatajobs().stream() + .map(DataJob::getUrn) + .collect(Collectors.toList()))) ); } @@ -1143,23 +913,16 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) { private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) { builder .type("DataFlow", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, - (env) -> ((DataFlow) env.getSource()).getPlatform().getUrn())) - ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final DataFlow dataFlow = env.getSource(); - return dataFlow.getDomain() != null ? dataFlow.getDomain().getUrn() : null; - }) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, + (env) -> ((DataFlow) env.getSource()).getPlatform().getUrn())) + .dataFetcher("domain", new LoadableTypeResolver<>( + domainType, + (env) -> { + final DataFlow dataFlow = env.getSource(); + return dataFlow.getDomain() != null ? dataFlow.getDomain().getUrn() : null; + }) ) ); } @@ -1170,69 +933,53 @@ private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) { private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builder) { builder .type("MLFeatureTable", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("platform", new AuthenticatedResolver<>( + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((MLFeatureTable) env.getSource()).getPlatform().getUrn())) - ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final MLFeatureTable entity = env.getSource(); - return entity.getDomain() != null ? entity.getDomain().getUrn() : null; - })) + .dataFetcher("domain", new LoadableTypeResolver<>( + domainType, + (env) -> { + final MLFeatureTable entity = env.getSource(); + return entity.getDomain() != null ? entity.getDomain().getUrn() : null; + })) ) .type("MLFeatureTableProperties", typeWiring -> typeWiring - .dataFetcher("mlFeatures", new AuthenticatedResolver<>( + .dataFetcher("mlFeatures", new LoadableTypeBatchResolver<>(mlFeatureType, (env) -> ((MLFeatureTableProperties) env.getSource()).getMlFeatures() != null ? ((MLFeatureTableProperties) env.getSource()).getMlFeatures().stream() .map(MLFeature::getUrn) .collect(Collectors.toList()) : ImmutableList.of())) - ) - .dataFetcher("mlPrimaryKeys", new AuthenticatedResolver<>( + .dataFetcher("mlPrimaryKeys", new LoadableTypeBatchResolver<>(mlPrimaryKeyType, (env) -> ((MLFeatureTableProperties) env.getSource()).getMlPrimaryKeys() != null ? ((MLFeatureTableProperties) env.getSource()).getMlPrimaryKeys().stream() .map(MLPrimaryKey::getUrn) .collect(Collectors.toList()) : ImmutableList.of())) - ) ) .type("MLFeatureProperties", typeWiring -> typeWiring - .dataFetcher("sources", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(datasetType, + .dataFetcher("sources", new LoadableTypeBatchResolver<>(datasetType, (env) -> ((MLFeatureProperties) env.getSource()).getSources().stream() .map(Dataset::getUrn) - .collect(Collectors.toList()))) + .collect(Collectors.toList())) ) ) .type("MLPrimaryKeyProperties", typeWiring -> typeWiring - .dataFetcher("sources", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(datasetType, + .dataFetcher("sources", new LoadableTypeBatchResolver<>(datasetType, (env) -> ((MLPrimaryKeyProperties) env.getSource()).getSources().stream() .map(Dataset::getUrn) - .collect(Collectors.toList()))) + .collect(Collectors.toList())) ) ) .type("MLModel", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, - (env) -> ((MLModel) env.getSource()).getPlatform().getUrn())) - ) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, + (env) -> ((MLModel) env.getSource()).getPlatform().getUrn())) .dataFetcher("domain", new LoadableTypeResolver<>( domainType, @@ -1242,8 +989,7 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde })) ) .type("MLModelProperties", typeWiring -> typeWiring - .dataFetcher("groups", new AuthenticatedResolver<>( - new LoadableTypeBatchResolver<>(mlModelGroupType, + .dataFetcher("groups", new LoadableTypeBatchResolver<>(mlModelGroupType, (env) -> { MLModelProperties properties = env.getSource(); if (properties.getGroups() != null) { @@ -1252,19 +998,14 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde .collect(Collectors.toList()); } return Collections.emptyList(); - })) + }) ) ) .type("MLModelGroup", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, - (env) -> ((MLModelGroup) env.getSource()).getPlatform().getUrn())) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, + (env) -> ((MLModelGroup) env.getSource()).getPlatform().getUrn()) ) .dataFetcher("domain", new LoadableTypeResolver<>( @@ -1275,12 +1016,8 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde })) ) .type("MLFeature", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) .dataFetcher("domain", new LoadableTypeResolver<>( domainType, @@ -1290,12 +1027,8 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde })) ) .type("MLPrimaryKey", typeWiring -> typeWiring - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) .dataFetcher("domain", new LoadableTypeResolver<>( domainType, @@ -1308,23 +1041,22 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde private void configureGlossaryRelationshipResolvers(final RuntimeWiring.Builder builder) { builder.type("GlossaryTerm", typeWiring -> typeWiring.dataFetcher("relationships", - new AuthenticatedResolver<>(new EntityRelationshipsResultResolver(graphClient)))); + new EntityRelationshipsResultResolver(graphClient))); } private void configureDomainResolvers(final RuntimeWiring.Builder builder) { builder.type("Domain", typeWiring -> typeWiring .dataFetcher("entities", new DomainEntitiesResolver(this.entityClient)) - .dataFetcher("relationships", new AuthenticatedResolver<>( - new EntityRelationshipsResultResolver(graphClient) - )) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient) + ) ); } private void configureAssertionResolvers(final RuntimeWiring.Builder builder) { builder.type("Assertion", typeWiring -> typeWiring.dataFetcher("relationships", - new AuthenticatedResolver<>(new EntityRelationshipsResultResolver(graphClient))) - .dataFetcher("platform", new AuthenticatedResolver<>( - new LoadableTypeResolver<>(dataPlatformType, (env) -> ((Assertion) env.getSource()).getPlatform().getUrn()))) + new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, + (env) -> ((Assertion) env.getSource()).getPlatform().getUrn())) .dataFetcher("runEvents", new AssertionRunEventResolver(entityClient))); } @@ -1348,23 +1080,20 @@ private void configurePolicyResolvers(final RuntimeWiring.Builder builder) { private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder builder) { builder.type("DataProcessInstance", typeWiring -> typeWiring - .dataFetcher("relationships", - new AuthenticatedResolver<>(new EntityRelationshipsResultResolver(graphClient))) - .dataFetcher("lineage", new AuthenticatedResolver<>( - new EntityLineageResultResolver(graphClient) - )) - .dataFetcher("state", new AuthenticatedResolver<>( + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("state", new TimeSeriesAspectResolver( this.entityClient, "dataProcessInstance", DATA_PROCESS_INSTANCE_RUN_EVENT_ASPECT_NAME, DataProcessInstanceRunEventMapper::map ) - )) + ) ); } - private DataLoader> createDataLoader(final LoadableType graphType, final QueryContext queryContext) { + private DataLoader> createDataLoader(final LoadableType graphType, final QueryContext queryContext) { BatchLoaderContextProvider contextProvider = () -> queryContext; DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider); return DataLoader.newDataLoader((keys, context) -> CompletableFuture.supplyAsync(() -> { @@ -1381,30 +1110,4 @@ private DataLoader> createDataLoader(final Load private void configureIngestionSourceResolvers(final RuntimeWiring.Builder builder) { builder.type("IngestionSource", typeWiring -> typeWiring.dataFetcher("executions", new IngestionSourceExecutionRequestsResolver(entityClient))); } - - private DataLoader> createAspectLoader(final QueryContext queryContext) { - BatchLoaderContextProvider contextProvider = () -> queryContext; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider); - return DataLoader.newDataLoader((keys, context) -> CompletableFuture.supplyAsync(() -> { - try { - log.debug(String.format("Batch loading aspects with keys: %s", keys)); - return aspectType.batchLoad(keys, context.getContext()); - } catch (Exception e) { - log.error(String.format("Failed to load Aspect for entity. keys: %s", keys) + " " + e.getMessage()); - throw new RuntimeException(String.format("Failed to retrieve entities of type Aspect", e)); - } - }), loaderOptions); - } - - private DataLoader> createUsageLoader(final QueryContext queryContext) { - BatchLoaderContextProvider contextProvider = () -> queryContext; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider); - return DataLoader.newDataLoader((keys, context) -> CompletableFuture.supplyAsync(() -> { - try { - return usageType.batchLoad(keys, context.getContext()); - } catch (Exception e) { - throw new RuntimeException(String.format("Failed to retrieve usage stats", e)); - } - }), loaderOptions); - } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/AuthenticatedResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/AuthenticatedResolver.java index 13556bdc5492f..2520b55c24e25 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/AuthenticatedResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/AuthenticatedResolver.java @@ -10,6 +10,7 @@ /** * Checks whether the user is currently authenticated & if so delegates execution to a child resolver. */ +@Deprecated public final class AuthenticatedResolver implements DataFetcher { private final DataFetcher _resolver; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowsePathsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowsePathsResolver.java index 2f736a8a5bdaf..4a1964b36032c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowsePathsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowsePathsResolver.java @@ -22,9 +22,9 @@ public class BrowsePathsResolver implements DataFetcher> _typeToEntity; + private final Map> _typeToEntity; - public BrowsePathsResolver(@Nonnull final List> browsableEntities) { + public BrowsePathsResolver(@Nonnull final List> browsableEntities) { _typeToEntity = browsableEntities.stream().collect(Collectors.toMap( BrowsableEntityType::type, entity -> entity diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseResolver.java index dc81c070a3b13..9c95eceb1e78f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseResolver.java @@ -24,9 +24,9 @@ public class BrowseResolver implements DataFetcher> _typeToEntity; + private final Map> _typeToEntity; - public BrowseResolver(@Nonnull final List> browsableEntities) { + public BrowseResolver(@Nonnull final List> browsableEntities) { _typeToEntity = browsableEntities.stream().collect(Collectors.toMap( BrowsableEntityType::type, entity -> entity diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java index 5c98190b736ba..90ca4b0a10811 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java @@ -22,11 +22,11 @@ */ public class EntityTypeBatchResolver implements DataFetcher>> { - private final List> _entityTypes; + private final List> _entityTypes; private final Function> _entitiesProvider; public EntityTypeBatchResolver( - final List> entityTypes, + final List> entityTypes, final Function> entitiesProvider ) { _entityTypes = entityTypes; @@ -40,7 +40,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) { return CompletableFuture.completedFuture(Collections.emptyList()); } // Assume all entities are of the same type - final com.linkedin.datahub.graphql.types.EntityType filteredEntity = + final com.linkedin.datahub.graphql.types.EntityType filteredEntity = Iterables.getOnlyElement(_entityTypes.stream() .filter(entity -> entities.get(0).getClass().isAssignableFrom(entity.objectClass())) .collect(Collectors.toList())); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java index e924d26c060a5..e1a5196e5f64f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java @@ -23,11 +23,11 @@ public class EntityTypeResolver implements DataFetcher> { private static final List IDENTITY_FIELDS = ImmutableList.of("__typename", "urn", "type"); - private final List> _entityTypes; + private final List> _entityTypes; private final Function _entityProvider; public EntityTypeResolver( - final List> entityTypes, + final List> entityTypes, final Function entity ) { _entityTypes = entityTypes; @@ -58,7 +58,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) { return CompletableFuture.completedFuture(javaObject); } - final com.linkedin.datahub.graphql.types.EntityType filteredEntity = Iterables.getOnlyElement(_entityTypes.stream() + final com.linkedin.datahub.graphql.types.EntityType filteredEntity = Iterables.getOnlyElement(_entityTypes.stream() .filter(entity -> javaObject.getClass().isAssignableFrom(entity.objectClass())) .collect(Collectors.toList())); final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(filteredEntity.name()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java index be932011fbb22..b9d5d406faa12 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java @@ -20,12 +20,12 @@ * * @param the generated GraphQL POJO corresponding to the resolved type. */ -public class LoadableTypeBatchResolver implements DataFetcher>> { +public class LoadableTypeBatchResolver implements DataFetcher>> { - private final LoadableType _loadableType; + private final LoadableType _loadableType; private final Function> _urnProvider; - public LoadableTypeBatchResolver(final LoadableType loadableType, final Function> urnProvider) { + public LoadableTypeBatchResolver(final LoadableType loadableType, final Function> urnProvider) { _loadableType = loadableType; _urnProvider = urnProvider; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java index 4d2fed1b7b03e..c260d11663549 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java @@ -19,23 +19,23 @@ * * @param the generated GraphQL POJO corresponding to the resolved type. */ -public class LoadableTypeResolver implements DataFetcher> { +public class LoadableTypeResolver implements DataFetcher> { - private final LoadableType _loadableType; - private final Function _urnProvider; + private final LoadableType _loadableType; + private final Function _keyProvider; - public LoadableTypeResolver(final LoadableType loadableType, final Function urnProvider) { + public LoadableTypeResolver(final LoadableType loadableType, final Function keyProvider) { _loadableType = loadableType; - _urnProvider = urnProvider; + _keyProvider = keyProvider; } @Override public CompletableFuture get(DataFetchingEnvironment environment) { - final String urn = _urnProvider.apply(environment); + final K urn = _keyProvider.apply(environment); if (urn == null) { return null; } - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(_loadableType.name()); + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(_loadableType.name()); return loader.load(urn); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/OwnerTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/OwnerTypeResolver.java index ebf7347869bb1..a4867819a2401 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/OwnerTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/OwnerTypeResolver.java @@ -25,10 +25,10 @@ */ public class OwnerTypeResolver implements DataFetcher> { - private final List> _loadableTypes; + private final List> _loadableTypes; private final Function _urnProvider; - public OwnerTypeResolver(final List> loadableTypes, final Function urnProvider) { + public OwnerTypeResolver(final List> loadableTypes, final Function urnProvider) { _loadableTypes = loadableTypes; _urnProvider = urnProvider; } @@ -36,7 +36,7 @@ public OwnerTypeResolver(final List> loadableTypes, final Functi @Override public CompletableFuture get(DataFetchingEnvironment environment) { final OwnerType ownerType = _urnProvider.apply(environment); - final LoadableType filteredEntity = Iterables.getOnlyElement(_loadableTypes.stream() + final LoadableType filteredEntity = Iterables.getOnlyElement(_loadableTypes.stream() .filter(entity -> ownerType.getClass().isAssignableFrom(entity.objectClass())) .collect(Collectors.toList())); final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(filteredEntity.name()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java index 911be3b9f2814..494f8be18c886 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java @@ -28,9 +28,9 @@ public class AutoCompleteForMultipleResolver implements DataFetcher> _typeToEntity; + private final Map> _typeToEntity; - public AutoCompleteForMultipleResolver(@Nonnull final List> searchableEntities) { + public AutoCompleteForMultipleResolver(@Nonnull final List> searchableEntities) { _typeToEntity = searchableEntities.stream().collect(Collectors.toMap( SearchableEntityType::type, entity -> entity diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java index aca1e808304f2..f594027fc2959 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java @@ -29,9 +29,9 @@ public class AutoCompleteResolver implements DataFetcher> _typeToEntity; + private final Map> _typeToEntity; - public AutoCompleteResolver(@Nonnull final List> searchableEntities) { + public AutoCompleteResolver(@Nonnull final List> searchableEntities) { _typeToEntity = searchableEntities.stream().collect(Collectors.toMap( SearchableEntityType::type, entity -> entity diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java index 99e81cefd99b7..608f3abaaa69f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java @@ -23,7 +23,7 @@ public class AutocompleteUtils { private AutocompleteUtils() { } public static CompletableFuture batchGetAutocompleteResults( - List> entities, + List> entities, String sanitizedQuery, AutoCompleteMultipleInput input, DataFetchingEnvironment environment diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/EntityInterfaceTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/EntityInterfaceTypeResolver.java index 46c899a2532aa..1a5f06da04014 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/EntityInterfaceTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/EntityInterfaceTypeResolver.java @@ -15,16 +15,16 @@ */ public class EntityInterfaceTypeResolver implements TypeResolver { - private final List> _entities; + private final List> _entities; - public EntityInterfaceTypeResolver(final List> entities) { + public EntityInterfaceTypeResolver(final List> entities) { _entities = entities; } @Override public GraphQLObjectType getType(TypeResolutionEnvironment env) { Object javaObject = env.getObject(); - final LoadableType filteredEntity = Iterables.getOnlyElement(_entities.stream() + final LoadableType filteredEntity = Iterables.getOnlyElement(_entities.stream() .filter(entity -> javaObject.getClass().isAssignableFrom(entity.objectClass())) .collect(Collectors.toList())); return env.getSchema().getObjectType(filteredEntity.objectClass().getSimpleName()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java index 036577a52339c..3f9e74d628882 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java @@ -15,7 +15,7 @@ * * @param : The GraphQL object type corresponding to the entity, must extend the `Entity` interface. */ -public interface BrowsableEntityType extends EntityType { +public interface BrowsableEntityType extends EntityType { /** * Retrieves {@link BrowseResults} corresponding to a given path, list of filters, start, & count. diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java index 3d677e77f3443..b6ce3e973a252 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java @@ -7,7 +7,7 @@ * * @param : The GraphQL object type corresponding to the entity, must be of type {@link Entity} */ -public interface EntityType extends LoadableType { +public interface EntityType extends LoadableType { /** * Retrieves the {@link com.linkedin.datahub.graphql.generated.EntityType} associated with the Graph type, eg. 'DATASET' diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java index 6ee0faedf6642..949a582978da7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java @@ -12,7 +12,7 @@ * * @param : The GraphQL object type corresponding to the type. */ -public interface LoadableType { +public interface LoadableType { /** * Returns generated GraphQL class associated with the type @@ -29,20 +29,20 @@ default String name() { /** * Retrieves an entity by urn string. Null is provided in place of an entity object if an entity cannot be found. * - * @param urn to retrieve + * @param key to retrieve * @param context the {@link QueryContext} corresponding to the request. */ - default DataFetcherResult load(@Nonnull final String urn, @Nonnull final QueryContext context) throws Exception { - return batchLoad(ImmutableList.of(urn), context).get(0); + default DataFetcherResult load(@Nonnull final K key, @Nonnull final QueryContext context) throws Exception { + return batchLoad(ImmutableList.of(key), context).get(0); }; /** * Retrieves an list of entities given a list of urn strings. The list returned is expected to * be of same length of the list of urns, where nulls are provided in place of an entity object if an entity cannot be found. * - * @param urns to retrieve + * @param keys to retrieve * @param context the {@link QueryContext} corresponding to the request. */ - List> batchLoad(@Nonnull final List urns, @Nonnull final QueryContext context) throws Exception; + List> batchLoad(@Nonnull final List keys, @Nonnull final QueryContext context) throws Exception; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/SearchableEntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/SearchableEntityType.java index 1da746c017c69..1740f80621c93 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/SearchableEntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/SearchableEntityType.java @@ -18,7 +18,7 @@ * @param : The GraphQL object type corresponding to the entity, must extend the `Entity` interface. */ @Deprecated -public interface SearchableEntityType extends EntityType { +public interface SearchableEntityType extends EntityType { /** * Deprecated - this is no longer used in favor of the search and searchAcrossEntities GraphQL resolver. diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java index 813d0b7a2726d..56ecdba9a68ab 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.graphql.VersionedAspectKey; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Aspect; +import com.linkedin.datahub.graphql.types.LoadableType; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.client.EntityClient; @@ -16,13 +17,23 @@ import javax.annotation.Nonnull; -public class AspectType { +public class AspectType implements LoadableType { private final EntityClient _entityClient; public AspectType(final EntityClient entityClient) { _entityClient = entityClient; } + @Override + public Class objectClass() { + return Aspect.class; + } + + @Override + public String name() { + return AspectType.class.getSimpleName(); + } + /** * Retrieves an list of aspects given a list of {@link VersionedAspectKey} structs. The list returned is expected to * be of same length of the list of keys, where nulls are provided in place of an aspect object if an entity cannot be found. diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java index ab7b32b5f7e0b..5fd3a02f6c436 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java @@ -19,7 +19,7 @@ import javax.annotation.Nonnull; -public class AssertionType implements com.linkedin.datahub.graphql.types.EntityType { +public class AssertionType implements com.linkedin.datahub.graphql.types.EntityType { static final Set ASPECTS_TO_FETCH = ImmutableSet.of( Constants.ASSERTION_KEY_ASPECT_NAME, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index e6de7b5e505af..17c09559161ca 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -54,7 +54,7 @@ import static com.linkedin.metadata.Constants.*; -public class ChartType implements SearchableEntityType, BrowsableEntityType, MutableType { +public class ChartType implements SearchableEntityType, BrowsableEntityType, MutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( CHART_KEY_ASPECT_NAME, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java index 6a31c3e5a22f2..f985183ad06ec 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java @@ -20,7 +20,7 @@ import javax.annotation.Nonnull; -public class ContainerType implements com.linkedin.datahub.graphql.types.EntityType { +public class ContainerType implements com.linkedin.datahub.graphql.types.EntityType { static final Set ASPECTS_TO_FETCH = ImmutableSet.of( Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java index 61e744bf93cec..e09f724cf8bf5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java @@ -42,7 +42,7 @@ import static com.linkedin.metadata.Constants.*; -public class CorpGroupType implements SearchableEntityType, MutableType { +public class CorpGroupType implements SearchableEntityType, MutableType { private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java index ffcf5cfcf0ae3..ec1f3a4889047 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java @@ -46,7 +46,7 @@ import static com.linkedin.metadata.Constants.*; -public class CorpUserType implements SearchableEntityType, MutableType { +public class CorpUserType implements SearchableEntityType, MutableType { private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index e6a3ca40d99fc..5d4faa8dec6bf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -55,7 +55,7 @@ import static com.linkedin.metadata.Constants.*; -public class DashboardType implements SearchableEntityType, BrowsableEntityType, +public class DashboardType implements SearchableEntityType, BrowsableEntityType, MutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index 13fb4fb9e3e4d..e14d3d206cb60 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -54,7 +54,7 @@ import static com.linkedin.metadata.Constants.*; -public class DataFlowType implements SearchableEntityType, BrowsableEntityType, +public class DataFlowType implements SearchableEntityType, BrowsableEntityType, MutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index 3ed5c49c9dcba..4ba73fcc4d987 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -54,7 +54,7 @@ import static com.linkedin.metadata.Constants.*; -public class DataJobType implements SearchableEntityType, BrowsableEntityType, +public class DataJobType implements SearchableEntityType, BrowsableEntityType, MutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java index 5ef3a5d8dce0e..c601602d6ab33 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java @@ -18,7 +18,7 @@ import static com.linkedin.metadata.Constants.*; -public class DataPlatformType implements EntityType { +public class DataPlatformType implements EntityType { private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 0d84ce5d61836..facacb19c7551 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -38,6 +38,7 @@ import com.linkedin.metadata.search.SearchResult; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.util.Pair; import graphql.execution.DataFetcherResult; import java.util.ArrayList; import java.util.Collection; @@ -53,7 +54,7 @@ import static com.linkedin.metadata.Constants.*; -public class DatasetType implements SearchableEntityType, BrowsableEntityType, +public class DatasetType implements SearchableEntityType>, BrowsableEntityType>, MutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( @@ -100,21 +101,19 @@ public EntityType type() { } @Override - public List> batchLoad(final List urnStrs, final QueryContext context) { - final List urns = urnStrs.stream() - .map(UrnUtils::getUrn) - .collect(Collectors.toList()); + public List> batchLoad(@Nonnull final List> versionedUrns, + @Nonnull final QueryContext context) { try { final Map datasetMap = - _entityClient.batchGetV2( + _entityClient.batchGetVersionedV2( Constants.DATASET_ENTITY_NAME, - new HashSet<>(urns), + new HashSet<>(versionedUrns), ASPECTS_TO_RESOLVE, context.getAuthentication()); final List gmsResults = new ArrayList<>(); - for (Urn urn : urns) { - gmsResults.add(datasetMap.getOrDefault(urn, null)); + for (Pair pair : versionedUrns) { + gmsResults.add(datasetMap.getOrDefault(UrnUtils.getUrn(pair.getFirst()), null)); } return gmsResults.stream() .map(gmsDataset -> gmsDataset == null ? null : DataFetcherResult.newResult() @@ -185,7 +184,7 @@ public Dataset update(@Nonnull String urn, @Nonnull DatasetUpdateInput input, @N throw new RuntimeException(String.format("Failed to write entity with urn %s", urn), e); } - return load(urn, context).getData(); + return load(new Pair<>(urn, null), context).getData(); } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java index 758b5da9e5e82..6df558ada3722 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java @@ -19,7 +19,7 @@ import javax.annotation.Nonnull; -public class DomainType implements com.linkedin.datahub.graphql.types.EntityType { +public class DomainType implements com.linkedin.datahub.graphql.types.EntityType { static final Set ASPECTS_TO_FETCH = ImmutableSet.of( Constants.DOMAIN_KEY_ASPECT_NAME, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java index 2be67ab39939c..2e9ae8883bee1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java @@ -38,7 +38,8 @@ import static com.linkedin.datahub.graphql.Constants.*; import static com.linkedin.metadata.Constants.*; -public class GlossaryTermType implements SearchableEntityType, BrowsableEntityType { +public class GlossaryTermType implements SearchableEntityType, + BrowsableEntityType { private static final Set FACET_FIELDS = ImmutableSet.of(""); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java index 45830117b9170..685f2aa084f98 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java @@ -38,7 +38,8 @@ import static com.linkedin.metadata.Constants.*; -public class MLFeatureTableType implements SearchableEntityType, BrowsableEntityType { +public class MLFeatureTableType implements SearchableEntityType, + BrowsableEntityType { private static final Set FACET_FIELDS = ImmutableSet.of("platform", "name"); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java index f4dffc4799467..577697aa66860 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java @@ -30,7 +30,7 @@ import static com.linkedin.metadata.Constants.*; -public class MLFeatureType implements SearchableEntityType { +public class MLFeatureType implements SearchableEntityType { private static final Set FACET_FIELDS = ImmutableSet.of(""); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java index f94f4649f0705..aa890604374eb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java @@ -38,7 +38,8 @@ import static com.linkedin.metadata.Constants.*; -public class MLModelGroupType implements SearchableEntityType, BrowsableEntityType { +public class MLModelGroupType implements SearchableEntityType, + BrowsableEntityType { private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java index 4342cee5e5496..375101dcea3d7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java @@ -38,7 +38,7 @@ import static com.linkedin.metadata.Constants.*; -public class MLModelType implements SearchableEntityType, BrowsableEntityType { +public class MLModelType implements SearchableEntityType, BrowsableEntityType { private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java index 5370dcf8fc398..ded6f33179e3e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java @@ -30,7 +30,7 @@ import static com.linkedin.metadata.Constants.*; -public class MLPrimaryKeyType implements SearchableEntityType { +public class MLPrimaryKeyType implements SearchableEntityType { private static final Set FACET_FIELDS = ImmutableSet.of(""); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java index 9a6844fb11d78..0735fab12f76b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java @@ -52,7 +52,8 @@ import static com.linkedin.datahub.graphql.Constants.*; import static com.linkedin.metadata.Constants.*; -public class NotebookType implements SearchableEntityType, BrowsableEntityType, MutableType { +public class NotebookType implements SearchableEntityType, BrowsableEntityType, + MutableType { static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( NOTEBOOK_KEY_ASPECT_NAME, NOTEBOOK_INFO_ASPECT_NAME, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java index 83e59d91e5afd..369f5911a1c9f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java @@ -43,7 +43,8 @@ import static com.linkedin.metadata.Constants.*; -public class TagType implements com.linkedin.datahub.graphql.types.SearchableEntityType, MutableType { +public class TagType implements com.linkedin.datahub.graphql.types.SearchableEntityType, + MutableType { private static final Set FACET_FIELDS = Collections.emptySet(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java index 658c8791a7771..745f3429f8e1b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java @@ -7,6 +7,7 @@ import com.linkedin.datahub.graphql.UsageStatsKey; import com.linkedin.datahub.graphql.VersionedAspectKey; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.types.LoadableType; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.usage.UsageClient; @@ -21,13 +22,23 @@ @Slf4j -public class UsageType { +public class UsageType implements LoadableType { private final UsageClient _usageClient; public UsageType(final UsageClient usageClient) { _usageClient = usageClient; } + @Override + public Class objectClass() { + return com.linkedin.datahub.graphql.generated.UsageQueryResult.class; + } + + @Override + public String name() { + return UsageType.class.getSimpleName(); + } + /** * Retrieves an list of aspects given a list of {@link VersionedAspectKey} structs. The list returned is expected to * be of same length of the list of keys, where nulls are provided in place of an aspect object if an entity cannot be found. diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 82c14baded6df..b10c4bb4acc5f 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -32,7 +32,7 @@ type Query { """ Fetch a Dataset by primary key (urn) """ - dataset(urn: String!): Dataset + dataset(urn: String!, versionStamp: String): Dataset """ Fetch a Dashboard by primary key (urn) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java index 4277426cf782d..79b1e4f345182 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java @@ -24,7 +24,7 @@ default String getIdentifier() { * @param entityName the name of the entity to be retrieved * @return an {@link DefaultEntitySpec} corresponding to the entity name provided, null if none exists. */ - @Nullable + @Nonnull EntitySpec getEntitySpec(@Nonnull final String entityName); /** diff --git a/li-utils/build.gradle b/li-utils/build.gradle index 8124f883c832a..a1fb0514c0d51 100644 --- a/li-utils/build.gradle +++ b/li-utils/build.gradle @@ -20,4 +20,4 @@ idea { } // Need to compile backing java definitions with the data template. -sourceSets.mainGeneratedDataTemplate.java.srcDirs('src/main/javaPegasus/') \ No newline at end of file +sourceSets.mainGeneratedDataTemplate.java.srcDirs('src/main/javaPegasus/', 'src/main/java/com/linkedin/util') \ No newline at end of file diff --git a/li-utils/src/main/java/com/linkedin/util/PairCoercer.java b/li-utils/src/main/java/com/linkedin/util/PairCoercer.java new file mode 100644 index 0000000000000..2787cce437b71 --- /dev/null +++ b/li-utils/src/main/java/com/linkedin/util/PairCoercer.java @@ -0,0 +1,24 @@ +package com.linkedin.util; + +import com.linkedin.data.template.Custom; +import com.linkedin.data.template.DirectCoercer; +import com.linkedin.data.template.TemplateOutputCastException; + + +public class PairCoercer implements DirectCoercer { + static { + Custom.registerCoercer(new PairCoercer(), Pair.class); + } + + @Override + public Object coerceInput(Pair object) throws ClassCastException { + return object.toString(); + } + + @Override + public Pair coerceOutput(Object object) throws TemplateOutputCastException { + String pairStr = (String) object; + String[] split = pairStr.split(","); + return Pair.of(split[0].substring(1), split[1].substring(0, split[1].length() - 1)); + } +} diff --git a/li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl b/li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl new file mode 100644 index 0000000000000..130b1bc2a72bc --- /dev/null +++ b/li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl @@ -0,0 +1,5 @@ +namespace com.linkedin.common + +@java.class = "com.linkedin.util.Pair" +@java.coercerClass = "com.linkedin.util.PairCoercer" +typeref Pair = string \ No newline at end of file diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index 9c7a1ae20ce38..df66ff0991d1f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -178,6 +178,24 @@ public Map getEntitiesV2( .collect(Collectors.toMap(Map.Entry::getKey, entry -> toEntityResponse(entry.getKey(), entry.getValue()))); } + /** + * Retrieves the aspects for the given set of urns and versions as dynamic aspect objects + * (Without having to define union objects) + * + * @param entityName name of the entity to fetch + * @param versionedUrnStrs set of urns to fetch with versions of aspects specified in a specialized string + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link Entity} object + */ + public Map getEntitiesVersionedV2( + @Nonnull final Set> versionedUrnStrs, + @Nonnull final Set aspectNames) throws URISyntaxException { + return getVersionedEnvelopedAspects(versionedUrnStrs, aspectNames) + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> toEntityResponse(entry.getKey(), entry.getValue()))); + } + /** * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects * @@ -191,6 +209,18 @@ public abstract Map> getLatestEnvelopedAspects( @Nonnull final Set urns, @Nonnull final Set aspectNames) throws URISyntaxException; + /** + * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects + * + * @param entityName name of the entity to fetch + * @param versionedUrnStrs set of urns to fetch with versions of aspects specified in a specialized string + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link EnvelopedAspect} object + */ + public abstract Map> getVersionedEnvelopedAspects( + @Nonnull final Set> versionedUrnStrs, + @Nonnull final Set aspectNames) throws URISyntaxException; + /** * Retrieves the latest aspect for the given urn as a list of enveloped aspects * diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index 66a6e59ede4fa..482e9996f8829 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -3,6 +3,8 @@ import com.linkedin.common.Status; import com.linkedin.common.urn.Urn; import com.linkedin.entity.EnvelopedAspect; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -28,4 +30,27 @@ public static boolean checkIfRemoved(EntityService entityService, Urn entityUrn) return false; } } + + public static Map convertVersionStamp(String versionStamp) { + String[] aspectNameVersionPairs = versionStamp.split(";"); + Map aspectVersionMap = new HashMap<>(); + for (String pair : aspectNameVersionPairs) { + String[] tokens = pair.split(":"); + if (tokens.length != 2) { + throw new IllegalArgumentException("Invalid version stamp cannot be parsed: " + versionStamp); + } + try { + aspectVersionMap.put(tokens[0], Long.valueOf(tokens[1])); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid value for aspect version: " + tokens[1]); + } + } + + return aspectVersionMap; + } + + public static String constructVersionStamp() { + //TODO: implement + return null; + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index fe3c6580491e7..5d8c61ec7cf13 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -20,6 +20,7 @@ import com.linkedin.metadata.aspect.Aspect; import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; import com.linkedin.metadata.entity.ListResult; import com.linkedin.metadata.entity.RollbackResult; import com.linkedin.metadata.entity.RollbackRunResult; @@ -217,6 +218,33 @@ public Map> getLatestEnvelopedAspects( .flatMap(List::stream) .collect(Collectors.toSet()); + return getCorrespondingAspects(dbKeys, urns); + } + + @Override + public Map> getVersionedEnvelopedAspects( + @Nonnull Set> versionedUrnStrs, + @Nonnull Set aspectNames) throws URISyntaxException { + + Map> urnAspectVersionMap = versionedUrnStrs.stream() + .collect(Collectors.toMap(Pair::getFirst, + pair -> EntityUtils.convertVersionStamp(pair.getSecond()))); + + final Set dbKeys = urnAspectVersionMap.entrySet().stream() + .map(entry -> aspectNames.stream() + .map(aspectName -> new EbeanAspectV2.PrimaryKey(entry.getKey(), aspectName, entry.getValue().get(aspectName))) + .collect(Collectors.toList())) + .flatMap(List::stream) + .collect(Collectors.toSet()); + + return getCorrespondingAspects(dbKeys, versionedUrnStrs.stream() + .map(Pair::getFirst) + .map(UrnUtils::getUrn).collect(Collectors.toSet())); + } + + private Map> getCorrespondingAspects(Set dbKeys, Set urns) + throws URISyntaxException { + final Map envelopedAspectMap = getEnvelopedAspects(dbKeys); // Group result by Urn @@ -225,7 +253,6 @@ public Map> getLatestEnvelopedAspects( .collect(Collectors.groupingBy(entry -> entry.getKey().getUrn(), Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); - // For each input urn, get the aspects corresponding to the urn. If empty, return the key aspect final Map> result = new HashMap<>(); for (Urn urn : urns) { List aspects = urnToAspects.getOrDefault(urn.toString(), Collections.emptyList()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java index 32ac981aedca6..054249b0704f4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java @@ -20,6 +20,7 @@ public class ChangeTransaction { List changeEvents; @ArraySchema(schema = @Schema(implementation = PatchOperation.class)) JsonPatch rawDiff; + String versionStamp; public void setSemanticVersion(String semanticVersion) { this.semVer = semanticVersion; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java index c0d4b03f700a3..ae97f989cf64a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java @@ -8,6 +8,9 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeline.data.ChangeCategory; import com.linkedin.metadata.timeline.data.ChangeEvent; @@ -55,10 +58,12 @@ public class EbeanTimelineService implements TimelineService { private final EbeanAspectDao _entityDao; //private final JacksonDataTemplateCodec _dataTemplateCodec = new JacksonDataTemplateCodec(); private final AspectDifferFactory _diffFactory; + private final EntityRegistry _entityRegistry; private final HashMap>> entityTypeElementAspectRegistry = new HashMap<>(); - public EbeanTimelineService(@Nonnull EbeanAspectDao entityDao) { + public EbeanTimelineService(@Nonnull EbeanAspectDao entityDao, @Nonnull EntityRegistry entityRegistry) { this._entityDao = entityDao; + _entityRegistry = entityRegistry; // TODO: Simplify this structure. // TODO: Load up from yaml file @@ -153,47 +158,60 @@ public List getTimeline(@Nonnull final Urn urn, @Nonnull fina startTimeMillis = endTimeMillis - DEFAULT_LOOKBACK_TIME_WINDOW_MILLIS; } - List foo = this._entityDao.getAspectsInRange(urn, aspectNames, startTimeMillis, endTimeMillis); + // TODO: We can pull full list of aspects here instead and pull each into version map for raw version stamp + EntitySpec entitySpec = _entityRegistry.getEntitySpec(urn.getEntityType()); + List aspectSpecs = entitySpec.getAspectSpecs(); + Set fullAspectNames = aspectSpecs.stream() + .filter(aspectSpec -> !aspectSpec.isTimeseries()) + .map(AspectSpec::getName) + .collect(Collectors.toSet()); + List aspectsInRange = this._entityDao.getAspectsInRange(urn, fullAspectNames, startTimeMillis, endTimeMillis); + //TODO: Prepopulate with all versioned aspectNames -> ignore timeseries using registry Map> aspectRowSetMap = new HashMap<>(); - foo.forEach(row -> { - TreeSet rowList = aspectRowSetMap.computeIfAbsent(row.getAspect(), - k -> new TreeSet<>(Comparator.comparing(EbeanAspectV2::getCreatedOn))); + fullAspectNames.forEach(aspectName -> + aspectRowSetMap.put(aspectName, new TreeSet<>(Comparator.comparing(EbeanAspectV2::getCreatedOn)))); + aspectsInRange.forEach(row -> { + TreeSet rowList = aspectRowSetMap.get(row.getAspect()); rowList.add(row); - /* - Long minVersion = aspectMinVersionMap.get(row.getAspect()); - if (minVersion == null) { - aspectMinVersionMap.put(row.getAspect(), row.getVersion()); - } else { - if (minVersion < row.getVersion() && minVersion != 0) { - aspectMinVersionMap.put(row.getAspect(), row.getVersion()); - } - } - */ }); // we need to pull previous versions of these aspects that are currently at a 0 - Map nextVersions = _entityDao.getNextVersions(urn.toString(), aspectNames); + Map nextVersions = _entityDao.getNextVersions(urn.toString(), fullAspectNames); for (Map.Entry> aspectMinVersion : aspectRowSetMap.entrySet()) { - EbeanAspectV2 oldestAspect = aspectMinVersion.getValue().first(); + //TODO: If empty, ignore this, but still do lookback + TreeSet aspectSet = aspectMinVersion.getValue(); + + EbeanAspectV2 oldestAspect = null; + if (!aspectSet.isEmpty()) { + oldestAspect = aspectMinVersion.getValue().first(); + } Long nextVersion = nextVersions.get(aspectMinVersion.getKey()); - if (((oldestAspect.getVersion() == 0L) && (nextVersion == 1L)) || (oldestAspect.getVersion() == 1L)) { - MissingEbeanAspectV2 missingEbeanAspectV2 = new MissingEbeanAspectV2(); - missingEbeanAspectV2.setAspect(aspectMinVersion.getKey()); - missingEbeanAspectV2.setCreatedOn(new Timestamp(0L)); - missingEbeanAspectV2.setVersion(-1); - aspectMinVersion.getValue().add(missingEbeanAspectV2); + // Fill out sentinel value if the oldest value possible has been retrieved, else get previous version prior to time range + if (oldestAspect != null && isOldestPossible(oldestAspect, nextVersion)) { + aspectMinVersion.getValue().add(createSentinel(aspectMinVersion.getKey())); } else { // get the next version - long versionToGet = (oldestAspect.getVersion() == 0L) ? nextVersion - 1 : oldestAspect.getVersion() - 1; + long versionToGet = 0; + if (oldestAspect != null) { + versionToGet = (oldestAspect.getVersion() == 0L) ? nextVersion - 1 : oldestAspect.getVersion() - 1; + } EbeanAspectV2 row = _entityDao.getAspect(urn.toString(), aspectMinVersion.getKey(), versionToGet); - aspectRowSetMap.get(row.getAspect()).add(row); + if (row != null) { + aspectRowSetMap.get(row.getAspect()).add(row); + } else { + aspectMinVersion.getValue().add(createSentinel(aspectMinVersion.getKey())); + } } } + //TODO: Process aspectRowSetMap and parse out versionStamps based on timestamps + // TODO: There are some extra steps happening here, we need to clean up how transactions get combined across differs - SortedMap> semanticDiffs = aspectRowSetMap.values() + SortedMap> semanticDiffs = aspectRowSetMap.entrySet() .stream() + .filter(entry -> aspectNames.contains(entry.getKey())) + .map(Map.Entry::getValue) .map(value -> computeDiffs(value, urn.getEntityType(), elementNames, rawDiffRequested)) .collect(TreeMap::new, this::combineComputedDiffsPerTransactionId, this::combineComputedDiffsPerTransactionId); // TODO:Move this down @@ -203,26 +221,26 @@ public List getTimeline(@Nonnull final Urn urn, @Nonnull fina List combinedChangeTransactions = combineTransactionsByTimestamp(changeTransactions); combinedChangeTransactions.sort(Comparator.comparing(ChangeTransaction::getTimestamp)); return combinedChangeTransactions; - /* - return foo.stream().map(row -> Model.SemanticChangeEvent.builder() - .changeType("UPSERT") - .target(urn.toString()) - .actor(null) - .actorId(null) - .description("This is a change at version " + row.getVersion()) - .element(row.getAspect()) - .elementId(null) - .proxy(null) - .rawDiff(row.getMetadata()) - .build()).collect(Collectors.toList()); - */ + } + + private boolean isOldestPossible(EbeanAspectV2 oldestAspect, long nextVersion) { + return (((oldestAspect.getVersion() == 0L) + && (nextVersion == 1L)) + || (oldestAspect.getVersion() == 1L)); + } + + private MissingEbeanAspectV2 createSentinel(String aspectName) { + MissingEbeanAspectV2 sentinel = new MissingEbeanAspectV2(); + sentinel.setAspect(aspectName); + sentinel.setCreatedOn(new Timestamp(0L)); + sentinel.setVersion(-1); + return sentinel; } private SortedMap> computeDiffs(TreeSet aspectTimeline, String entityType, Set elementNames, boolean rawDiffsRequested) { EbeanAspectV2 previousValue = null; SortedMap> changeTransactionsMap = new TreeMap<>(); - //long transactionId = FIRST_TRANSACTION_ID; long transactionId; for (EbeanAspectV2 currentValue : aspectTimeline) { transactionId = currentValue.getCreatedOn().getTime(); @@ -230,7 +248,6 @@ private SortedMap> computeDiffs(TreeSet batchGetV2( @Nullable final Set aspectNames, @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException; + @Nonnull + Map batchGetVersionedV2( + @Nonnull String entityName, + @Nonnull final Set> versionedUrns, + @Nullable final Set aspectNames, + @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException; + @Nonnull @Deprecated public Map batchGet(@Nonnull final Set urns, @Nonnull final Authentication authentication) diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java index f805be33c7186..e408c5b78ad96 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java @@ -37,6 +37,7 @@ import com.linkedin.mxe.PlatformEvent; import com.linkedin.mxe.SystemMetadata; import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.util.Pair; import io.opentelemetry.extension.annotations.WithSpan; import java.net.URISyntaxException; import java.time.Clock; @@ -98,6 +99,18 @@ public Map batchGetV2( return _entityService.getEntitiesV2(entityName, urns, projectedAspects); } + @Nonnull + public Map batchGetVersionedV2( + @Nonnull String entityName, + @Nonnull final Set> versionedUrns, + @Nullable final Set aspectNames, + @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException { + final Set projectedAspects = aspectNames == null + ? _entityService.getEntityAspectNames(entityName) + : aspectNames; + return _entityService.getEntitiesVersionedV2(versionedUrns, projectedAspects); + } + @Nonnull public Map batchGet(@Nonnull final Set urns, @Nonnull final Authentication authentication) { return _entityService.getEntities(urns, ImmutableSet.of()); diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index b39af3bfc20d0..b09f240847524 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -30,6 +30,8 @@ import com.linkedin.entity.EntitiesV2BatchGetRequestBuilder; import com.linkedin.entity.EntitiesV2GetRequestBuilder; import com.linkedin.entity.EntitiesV2RequestBuilders; +import com.linkedin.entity.EntitiesVersionedV2BatchGetRequestBuilder; +import com.linkedin.entity.EntitiesVersionedV2RequestBuilders; import com.linkedin.entity.Entity; import com.linkedin.entity.EntityArray; import com.linkedin.entity.EntityResponse; @@ -53,6 +55,8 @@ import com.linkedin.restli.client.Client; import com.linkedin.restli.client.RestLiResponseException; import com.linkedin.restli.common.HttpStatus; +import com.linkedin.util.Pair; +import com.linkedin.util.PairCoercer; import java.net.URISyntaxException; import java.util.Collection; import java.util.HashMap; @@ -76,6 +80,8 @@ public class RestliEntityClient extends BaseClient implements EntityClient { private static final EntitiesRequestBuilders ENTITIES_REQUEST_BUILDERS = new EntitiesRequestBuilders(); private static final EntitiesV2RequestBuilders ENTITIES_V2_REQUEST_BUILDERS = new EntitiesV2RequestBuilders(); + private static final EntitiesVersionedV2RequestBuilders ENTITIES_VERSIONED_V2_REQUEST_BUILDERS = + new EntitiesVersionedV2RequestBuilders(); private static final AspectsRequestBuilders ASPECTS_REQUEST_BUILDERS = new AspectsRequestBuilders(); private static final PlatformRequestBuilders PLATFORM_REQUEST_BUILDERS = new PlatformRequestBuilders(); @@ -171,6 +177,44 @@ public Map batchGetV2(@Nonnull String entityName, @Nonnull }, entry -> entry.getValue().getEntity())); } + /** + * Batch get a set of versioned aspects for a single entity. + * + * @param entityName the entity type to fetch + * @param versionedUrns the urns of the entities to batch get + * @param aspectNames the aspect names to batch get + * @param authentication the authentication to include in the request to the Metadata Service + * @throws RemoteInvocationException + */ + @Nonnull + public Map batchGetVersionedV2( + @Nonnull String entityName, + @Nonnull final Set> versionedUrns, + @Nullable final Set aspectNames, + @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException { + + final EntitiesVersionedV2BatchGetRequestBuilder requestBuilder = ENTITIES_VERSIONED_V2_REQUEST_BUILDERS.batchGet() + .aspectsParam(aspectNames) + .entityTypeParam(entityName) + .ids(versionedUrns.stream() + .map(Pair::toString) + .map(string -> new PairCoercer().coerceOutput(string)).collect( + Collectors.toSet())); + + return sendClientRequest(requestBuilder, authentication).getEntity() + .getResults() + .entrySet() + .stream() + .collect(Collectors.toMap(entry -> { + try { + return Urn.createFromString((String) entry.getKey().getFirst()); + } catch (URISyntaxException e) { + throw new RuntimeException( + String.format("Failed to bind urn string with value %s into urn", entry.getKey())); + } + }, entry -> entry.getValue().getEntity())); + } + /** * Autocomplete a search query for a particular field of an entity. * diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java index 5ba91ee2df275..be331a453f410 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java @@ -24,7 +24,8 @@ import javax.inject.Named; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.resources.restli.RestliConstants.PARAM_ASPECTS; +import static com.linkedin.metadata.resources.entity.ResourceUtils.*; +import static com.linkedin.metadata.resources.restli.RestliConstants.*; import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; @@ -52,7 +53,7 @@ public Task get(@Nonnull String urnStr, return RestliUtil.toTask(() -> { final String entityName = urnToEntityName(urn); final Set projectedAspects = - aspectNames == null ? getAllAspectNames(entityName) : new HashSet<>(Arrays.asList(aspectNames)); + aspectNames == null ? getAllAspectNames(_entityService, entityName) : new HashSet<>(Arrays.asList(aspectNames)); try { return _entityService.getEntityV2(entityName, urn, projectedAspects); } catch (Exception e) { @@ -78,7 +79,7 @@ public Task> batchGet(@Nonnull Set urnStrs, final String entityName = urnToEntityName(urns.iterator().next()); return RestliUtil.toTask(() -> { final Set projectedAspects = - aspectNames == null ? getAllAspectNames(entityName) : new HashSet<>(Arrays.asList(aspectNames)); + aspectNames == null ? getAllAspectNames(_entityService, entityName) : new HashSet<>(Arrays.asList(aspectNames)); try { return _entityService.getEntitiesV2(entityName, urns, projectedAspects); } catch (Exception e) { @@ -88,8 +89,4 @@ public Task> batchGet(@Nonnull Set urnStrs, } }, MetricRegistry.name(this.getClass(), "batchGet")); } - - private Set getAllAspectNames(final String entityName) { - return _entityService.getEntityAspectNames(entityName); - } } \ No newline at end of file diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java new file mode 100644 index 0000000000000..606fb6b438d4a --- /dev/null +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java @@ -0,0 +1,66 @@ +package com.linkedin.metadata.resources.entity; + +import com.codahale.metrics.MetricRegistry; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.EntityResponse; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.restli.RestliUtil; +import com.linkedin.parseq.Task; +import com.linkedin.restli.server.annotations.Optional; +import com.linkedin.restli.server.annotations.QueryParam; +import com.linkedin.restli.server.annotations.RestLiCollection; +import com.linkedin.restli.server.annotations.RestMethod; +import com.linkedin.restli.server.resources.CollectionResourceTaskTemplate; +import com.linkedin.util.Pair; +import io.opentelemetry.extension.annotations.WithSpan; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.inject.Named; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.resources.entity.ResourceUtils.*; +import static com.linkedin.metadata.resources.restli.RestliConstants.*; + + +/** + * Single unified resource for fetching, updating, searching, & browsing versioned DataHub entities + */ +@Slf4j +@RestLiCollection(name = "entitiesVersionedV2", namespace = "com.linkedin.entity", + keyTyperefClass = com.linkedin.common.Pair.class) +public class EntityVersionedV2Resource extends CollectionResourceTaskTemplate, EntityResponse> { + + @Inject + @Named("entityService") + private EntityService _entityService; + + @RestMethod.BatchGet + @Nonnull + @WithSpan + public Task> batchGetVersioned( + @Nonnull Set> versionedUrnStrs, + @QueryParam(PARAM_ENTITY_TYPE) @Nonnull String entityType, + @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) { + log.debug("BATCH GET VERSIONED V2 {}", versionedUrnStrs); + if (versionedUrnStrs.size() <= 0) { + return Task.value(Collections.emptyMap()); + } + return RestliUtil.toTask(() -> { + final Set projectedAspects = + aspectNames == null ? getAllAspectNames(_entityService, entityType) : new HashSet<>(Arrays.asList(aspectNames)); + try { + return _entityService.getEntitiesVersionedV2(versionedUrnStrs, projectedAspects); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to batch get versioned entities: %s, projectedAspects: %s", versionedUrnStrs, projectedAspects), + e); + } + }, MetricRegistry.name(this.getClass(), "batchGet")); + } +} diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java new file mode 100644 index 0000000000000..679cb421bc5fe --- /dev/null +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java @@ -0,0 +1,15 @@ +package com.linkedin.metadata.resources.entity; + +import com.linkedin.metadata.entity.EntityService; +import java.util.Set; + + +public class ResourceUtils { + private ResourceUtils() { + + } + + public static Set getAllAspectNames(final EntityService entityService, final String entityName) { + return entityService.getEntityAspectNames(entityName); + } +} diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java index e161779900bda..1100d5a194928 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java @@ -34,4 +34,6 @@ private RestliConstants() { } public static final String PARAM_URNS = "urns"; public static final String PARAM_MODE = "mode"; public static final String PARAM_DIRECTION = "direction"; + public static final String PARAM_ENTITY_TYPE = "entityType"; + public static final String PARAM_VERSIONED_URN_PAIRS = "versionedUrns"; } From 31106428df7fffaa873f34f791905bc9f1e6548a Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 21 Apr 2022 15:35:55 -0500 Subject: [PATCH 02/21] feat(versionedDataset): cover null versionStamp --- .../java/com/linkedin/metadata/entity/EntityUtils.java | 7 ++++++- .../linkedin/metadata/entity/ebean/EbeanEntityService.java | 3 ++- .../metadata/timeline/ebean/EbeanTimelineService.java | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index 482e9996f8829..8b32b4e846fc4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -5,7 +5,9 @@ import com.linkedin.entity.EnvelopedAspect; import java.util.HashMap; import java.util.Map; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; @Slf4j @@ -32,8 +34,11 @@ public static boolean checkIfRemoved(EntityService entityService, Urn entityUrn) } public static Map convertVersionStamp(String versionStamp) { - String[] aspectNameVersionPairs = versionStamp.split(";"); Map aspectVersionMap = new HashMap<>(); + if (StringUtils.isBlank(versionStamp)) { + return aspectVersionMap; + } + String[] aspectNameVersionPairs = versionStamp.split(";"); for (String pair : aspectNameVersionPairs) { String[] tokens = pair.split(":"); if (tokens.length != 2) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 5d8c61ec7cf13..365d31ffa4171 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -232,7 +232,8 @@ public Map> getVersionedEnvelopedAspects( final Set dbKeys = urnAspectVersionMap.entrySet().stream() .map(entry -> aspectNames.stream() - .map(aspectName -> new EbeanAspectV2.PrimaryKey(entry.getKey(), aspectName, entry.getValue().get(aspectName))) + .map(aspectName -> new EbeanAspectV2.PrimaryKey(entry.getKey(), aspectName, + entry.getValue().getOrDefault(aspectName, 0L))) .collect(Collectors.toList())) .flatMap(List::stream) .collect(Collectors.toSet()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java index ae97f989cf64a..5b4faa9870943 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java @@ -179,7 +179,6 @@ public List getTimeline(@Nonnull final Urn urn, @Nonnull fina Map nextVersions = _entityDao.getNextVersions(urn.toString(), fullAspectNames); for (Map.Entry> aspectMinVersion : aspectRowSetMap.entrySet()) { - //TODO: If empty, ignore this, but still do lookback TreeSet aspectSet = aspectMinVersion.getValue(); EbeanAspectV2 oldestAspect = null; From c5cd6c65ac8b9946e128f473d4d650cb2ce19bbc Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 21 Apr 2022 19:27:16 -0500 Subject: [PATCH 03/21] feat(versionedDataset): support returning versionStamp from the backend --- .../linkedin/metadata/entity/EntityUtils.java | 14 ++- .../entity/ebean/EbeanEntityService.java | 2 +- .../timeline/data/ChangeTransaction.java | 2 + .../timeline/ebean/EbeanTimelineService.java | 103 ++++++++++++++---- 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index 8b32b4e846fc4..07271e12bda77 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -5,7 +5,7 @@ import com.linkedin.entity.EnvelopedAspect; import java.util.HashMap; import java.util.Map; -import javax.annotation.Nullable; +import java.util.SortedMap; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; @@ -34,7 +34,7 @@ public static boolean checkIfRemoved(EntityService entityService, Urn entityUrn) } public static Map convertVersionStamp(String versionStamp) { - Map aspectVersionMap = new HashMap<>(); + Map aspectVersionMap = new HashMap<>(); if (StringUtils.isBlank(versionStamp)) { return aspectVersionMap; } @@ -54,8 +54,12 @@ public static Map convertVersionStamp(String versionStamp) { return aspectVersionMap; } - public static String constructVersionStamp() { - //TODO: implement - return null; + public static String constructVersionStamp(SortedMap versionStampMap) { + StringBuilder versionStamp = versionStampMap.entrySet().stream() + .collect(StringBuilder::new, (builder, entry) -> builder.append(entry.getKey()) + .append(":") + .append(entry.getValue()).append(";"), StringBuilder::append); + // trim off last ; + return versionStamp.substring(0, versionStamp.length() - 1); } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 365d31ffa4171..727a9c0f9d843 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -232,7 +232,7 @@ public Map> getVersionedEnvelopedAspects( final Set dbKeys = urnAspectVersionMap.entrySet().stream() .map(entry -> aspectNames.stream() - .map(aspectName -> new EbeanAspectV2.PrimaryKey(entry.getKey(), aspectName, + .map(aspectName -> new EbeanAspectV2.PrimaryKey(entry.getKey(), aspectName, entry.getValue().getOrDefault(aspectName, 0L))) .collect(Collectors.toList())) .flatMap(List::stream) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java index 054249b0704f4..5037b8dde9a8b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java @@ -6,6 +6,7 @@ import java.util.List; import lombok.Builder; import lombok.Getter; +import lombok.Setter; @Getter @@ -20,6 +21,7 @@ public class ChangeTransaction { List changeEvents; @ArraySchema(schema = @Schema(implementation = PatchOperation.class)) JsonPatch rawDiff; + @Setter String versionStamp; public void setSemanticVersion(String semanticVersion) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java index 5b4faa9870943..3ab2648fa2e7f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java @@ -33,6 +33,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -45,6 +46,7 @@ import org.apache.parquet.SemanticVersion; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.entity.EntityUtils.*; public class EbeanTimelineService implements TimelineService { @@ -158,7 +160,7 @@ public List getTimeline(@Nonnull final Urn urn, @Nonnull fina startTimeMillis = endTimeMillis - DEFAULT_LOOKBACK_TIME_WINDOW_MILLIS; } - // TODO: We can pull full list of aspects here instead and pull each into version map for raw version stamp + // Pull full list of aspects for entity and filter timeseries aspects for range query EntitySpec entitySpec = _entityRegistry.getEntitySpec(urn.getEntityType()); List aspectSpecs = entitySpec.getAspectSpecs(); Set fullAspectNames = aspectSpecs.stream() @@ -166,7 +168,40 @@ public List getTimeline(@Nonnull final Urn urn, @Nonnull fina .map(AspectSpec::getName) .collect(Collectors.toSet()); List aspectsInRange = this._entityDao.getAspectsInRange(urn, fullAspectNames, startTimeMillis, endTimeMillis); - //TODO: Prepopulate with all versioned aspectNames -> ignore timeseries using registry + + // Prepopulate with all versioned aspectNames -> ignore timeseries using registry + Map> aspectRowSetMap = constructAspectRowSetMap(urn, fullAspectNames, aspectsInRange); + + Map> timestampVersionCache = constructTimestampVersionCache(aspectRowSetMap); + + // TODO: There are some extra steps happening here, we need to clean up how transactions get combined across differs + SortedMap> semanticDiffs = aspectRowSetMap.entrySet() + .stream() + .filter(entry -> aspectNames.contains(entry.getKey())) + .map(Map.Entry::getValue) + .map(value -> computeDiffs(value, urn.getEntityType(), elementNames, rawDiffRequested)) + .collect(TreeMap::new, this::combineComputedDiffsPerTransactionId, this::combineComputedDiffsPerTransactionId); + // TODO:Move this down + assignSemanticVersions(semanticDiffs); + List changeTransactions = + semanticDiffs.values().stream().collect(ArrayList::new, ArrayList::addAll, ArrayList::addAll); + List combinedChangeTransactions = combineTransactionsByTimestamp(changeTransactions, + timestampVersionCache); + combinedChangeTransactions.sort(Comparator.comparing(ChangeTransaction::getTimestamp)); + return combinedChangeTransactions; + } + + /** + * Constructs a map from aspect name to a sorted set of DB aspects by created timestamp. Set includes all aspects + * relevant to an entity and does a lookback by 1 for all aspects, creating sentinel values for when the oldest aspect + * possible has been retrieved or no value exists in the DB for an aspect + * @param urn urn of the entity + * @param fullAspectNames full list of aspects relevant to the entity + * @param aspectsInRange aspects returned by the range query by timestampm + * @return map constructed as described + */ + private Map> constructAspectRowSetMap(Urn urn, Set fullAspectNames, + List aspectsInRange) { Map> aspectRowSetMap = new HashMap<>(); fullAspectNames.forEach(aspectName -> aspectRowSetMap.put(aspectName, new TreeSet<>(Comparator.comparing(EbeanAspectV2::getCreatedOn)))); @@ -203,23 +238,7 @@ public List getTimeline(@Nonnull final Urn urn, @Nonnull fina } } } - - //TODO: Process aspectRowSetMap and parse out versionStamps based on timestamps - - // TODO: There are some extra steps happening here, we need to clean up how transactions get combined across differs - SortedMap> semanticDiffs = aspectRowSetMap.entrySet() - .stream() - .filter(entry -> aspectNames.contains(entry.getKey())) - .map(Map.Entry::getValue) - .map(value -> computeDiffs(value, urn.getEntityType(), elementNames, rawDiffRequested)) - .collect(TreeMap::new, this::combineComputedDiffsPerTransactionId, this::combineComputedDiffsPerTransactionId); - // TODO:Move this down - assignSemanticVersions(semanticDiffs); - List changeTransactions = - semanticDiffs.values().stream().collect(ArrayList::new, ArrayList::addAll, ArrayList::addAll); - List combinedChangeTransactions = combineTransactionsByTimestamp(changeTransactions); - combinedChangeTransactions.sort(Comparator.comparing(ChangeTransaction::getTimestamp)); - return combinedChangeTransactions; + return aspectRowSetMap; } private boolean isOldestPossible(EbeanAspectV2 oldestAspect, long nextVersion) { @@ -236,6 +255,48 @@ private MissingEbeanAspectV2 createSentinel(String aspectName) { return sentinel; } + /** + * Constructs a map from timestamp to a sorted map of aspect name -> version for use in constructing the version stamp + * @param aspectRowSetMap map constructed as described in {@link EbeanTimelineService#constructAspectRowSetMap} + * @return map as described + */ + private Map> constructTimestampVersionCache(Map> aspectRowSetMap) { + Set aspects = aspectRowSetMap.values().stream() + .flatMap(TreeSet::stream) + .filter(aspect -> aspect.getVersion() != -1L) + .collect(Collectors.toSet()); + Map> timestampVersionCache = new HashMap<>(); + for (EbeanAspectV2 aspect : aspects) { + if (timestampVersionCache.containsKey(aspect.getCreatedOn().getTime())) { + continue; + } + SortedMap versionStampMap = new TreeMap<>(Comparator.naturalOrder()); + for (TreeSet aspectSet : aspectRowSetMap.values()) { + EbeanAspectV2 maybeMatch = null; + for (EbeanAspectV2 aspect2 : aspectSet) { + if (aspect2 instanceof MissingEbeanAspectV2) { + continue; + } + if (aspect.getCreatedOn().getTime() < aspect2.getCreatedOn().getTime()) { + // Can't match a higher timestamp, fall back to maybeMatch or empty + break; + } + if (aspect.getCreatedOn().getTime() == aspect2.getCreatedOn().getTime()) { + versionStampMap.put(aspect2.getAspect(), aspect2.getVersion()); + break; + } else { + maybeMatch = aspect2; + } + } + if (maybeMatch != null) { + versionStampMap.put(maybeMatch.getAspect(), maybeMatch.getVersion()); + } + } + timestampVersionCache.put(aspect.getCreatedOn().getTime(), versionStampMap); + } + return timestampVersionCache; + } + private SortedMap> computeDiffs(TreeSet aspectTimeline, String entityType, Set elementNames, boolean rawDiffsRequested) { EbeanAspectV2 previousValue = null; @@ -347,7 +408,8 @@ private SemanticVersion getGroupSemanticVersion(SemanticChangeType highestChange return previousVersion; } - private List combineTransactionsByTimestamp(List changeTransactions) { + private List combineTransactionsByTimestamp(List changeTransactions, Map> timestampVersionCache) { Map> transactionsByTimestamp = changeTransactions.stream().collect(Collectors.groupingBy(ChangeTransaction::getTimestamp)); List combinedChangeTransactions = new ArrayList<>(); @@ -366,6 +428,7 @@ private List combineTransactionsByTimestamp(List Date: Thu, 21 Apr 2022 22:49:44 -0500 Subject: [PATCH 04/21] feat(versionedDataset): fix graphql and timeline cli --- .../datahub/graphql/GmsGraphQLEngine.java | 43 +++++++------------ .../load/EntityTypeBatchResolver.java | 13 ++++-- .../resolvers/load/EntityTypeResolver.java | 9 ++-- .../load/LoadableTypeBatchResolver.java | 14 +++--- .../resolvers/load/LoadableTypeResolver.java | 6 +-- .../datahub/graphql/types/EntityType.java | 4 ++ .../types/assertion/AssertionType.java | 7 +++ .../graphql/types/chart/ChartType.java | 7 +++ .../types/container/ContainerType.java | 7 +++ .../types/corpgroup/CorpGroupType.java | 7 +++ .../graphql/types/corpuser/CorpUserType.java | 7 +++ .../types/dashboard/DashboardType.java | 7 +++ .../graphql/types/dataflow/DataFlowType.java | 7 +++ .../graphql/types/datajob/DataJobType.java | 7 +++ .../types/dataplatform/DataPlatformType.java | 7 +++ .../graphql/types/dataset/DatasetType.java | 7 +++ .../graphql/types/domain/DomainType.java | 7 +++ .../types/glossary/GlossaryTermType.java | 7 +++ .../types/mlmodel/MLFeatureTableType.java | 7 +++ .../graphql/types/mlmodel/MLFeatureType.java | 7 +++ .../types/mlmodel/MLModelGroupType.java | 7 +++ .../graphql/types/mlmodel/MLModelType.java | 7 +++ .../types/mlmodel/MLPrimaryKeyType.java | 7 +++ .../graphql/types/notebook/NotebookType.java | 7 +++ .../datahub/graphql/types/tag/TagType.java | 7 +++ .../src/datahub/cli/timeline_cli.py | 8 ++-- .../metadata/entity/EntityService.java | 2 - 27 files changed, 181 insertions(+), 51 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index d48ff270d64fe..7e4d6a5870640 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -571,43 +571,35 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder) { builder .type("SearchResult", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - entityTypes.stream().collect(Collectors.toList()), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((SearchResult) env.getSource()).getEntity())) ) .type("SearchAcrossLineageResult", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - entityTypes.stream().collect(Collectors.toList()), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((SearchAcrossLineageResult) env.getSource()).getEntity())) ) .type("AggregationMetadata", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - entityTypes.stream().collect(Collectors.toList()), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((AggregationMetadata) env.getSource()).getEntity())) ) .type("RecommendationContent", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - entityTypes.stream().collect(Collectors.toList()), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((RecommendationContent) env.getSource()).getEntity())) ) .type("BrowseResults", typeWiring -> typeWiring - .dataFetcher("entities", new EntityTypeBatchResolver( - entityTypes.stream().collect(Collectors.toList()), + .dataFetcher("entities", new EntityTypeBatchResolver(entityTypes, (env) -> ((BrowseResults) env.getSource()).getEntities())) ) .type("EntityRelationshipLegacy", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - new ArrayList<>(entityTypes), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((EntityRelationshipLegacy) env.getSource()).getEntity())) ) .type("EntityRelationship", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - new ArrayList<>(entityTypes), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((EntityRelationship) env.getSource()).getEntity())) ) .type("LineageRelationship", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - new ArrayList<>(entityTypes), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((LineageRelationship) env.getSource()).getEntity())) ) .type("ListDomainsResult", typeWiring -> typeWiring @@ -618,18 +610,15 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder ) .type("AutoCompleteResults", typeWiring -> typeWiring .dataFetcher("entities", - new EntityTypeBatchResolver( - new ArrayList<>(entityTypes), + new EntityTypeBatchResolver(entityTypes, (env) -> ((AutoCompleteResults) env.getSource()).getEntities())) ) .type("AutoCompleteResultForEntity", typeWiring -> typeWiring - .dataFetcher("entities", new EntityTypeBatchResolver( - new ArrayList<>(entityTypes), + .dataFetcher("entities", new EntityTypeBatchResolver(entityTypes, (env) -> ((AutoCompleteResultForEntity) env.getSource()).getEntities())) ) .type("PolicyMatchCriterionValue", typeWiring -> typeWiring - .dataFetcher("entity", new EntityTypeResolver( - new ArrayList<>(entityTypes), + .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((PolicyMatchCriterionValue) env.getSource()).getEntity())) ); } @@ -824,7 +813,7 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) { builder.type("ChartInfo", typeWiring -> typeWiring .dataFetcher("inputs", new LoadableTypeBatchResolver<>(datasetType, (env) -> ((ChartInfo) env.getSource()).getInputs().stream() - .map(Dataset::getUrn) + .map(datasetType.getKeyProvider()) .collect(Collectors.toList()))) ); } @@ -894,11 +883,11 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) { .type("DataJobInputOutput", typeWiring -> typeWiring .dataFetcher("inputDatasets", new LoadableTypeBatchResolver<>(datasetType, (env) -> ((DataJobInputOutput) env.getSource()).getInputDatasets().stream() - .map(Dataset::getUrn) + .map(datasetType.getKeyProvider()) .collect(Collectors.toList()))) .dataFetcher("outputDatasets", new LoadableTypeBatchResolver<>(datasetType, (env) -> ((DataJobInputOutput) env.getSource()).getOutputDatasets().stream() - .map(Dataset::getUrn) + .map(datasetType.getKeyProvider()) .collect(Collectors.toList()))) .dataFetcher("inputDatajobs", new LoadableTypeBatchResolver<>(dataJobType, (env) -> ((DataJobInputOutput) env.getSource()).getInputDatajobs().stream() @@ -964,14 +953,14 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde .type("MLFeatureProperties", typeWiring -> typeWiring .dataFetcher("sources", new LoadableTypeBatchResolver<>(datasetType, (env) -> ((MLFeatureProperties) env.getSource()).getSources().stream() - .map(Dataset::getUrn) + .map(datasetType.getKeyProvider()) .collect(Collectors.toList())) ) ) .type("MLPrimaryKeyProperties", typeWiring -> typeWiring .dataFetcher("sources", new LoadableTypeBatchResolver<>(datasetType, (env) -> ((MLPrimaryKeyProperties) env.getSource()).getSources().stream() - .map(Dataset::getUrn) + .map(datasetType.getKeyProvider()) .collect(Collectors.toList())) ) ) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java index 90ca4b0a10811..0643f86dda265 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.graphql.generated.Entity; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -34,18 +35,22 @@ public EntityTypeBatchResolver( } @Override - public CompletableFuture get(DataFetchingEnvironment environment) { + public CompletableFuture> get(DataFetchingEnvironment environment) { final List entities = _entitiesProvider.apply(environment); if (entities.isEmpty()) { return CompletableFuture.completedFuture(Collections.emptyList()); } // Assume all entities are of the same type - final com.linkedin.datahub.graphql.types.EntityType filteredEntity = + final com.linkedin.datahub.graphql.types.EntityType filteredEntity = Iterables.getOnlyElement(_entityTypes.stream() .filter(entity -> entities.get(0).getClass().isAssignableFrom(entity.objectClass())) .collect(Collectors.toList())); - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(filteredEntity.name()); - return loader.loadMany(entities.stream().map(Entity::getUrn).collect(Collectors.toList())); + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(filteredEntity.name()); + List keyList = new ArrayList(); + for (Entity entity : entities) { + keyList.add(filteredEntity.getKeyProvider().apply(entity)); + } + return loader.loadMany(keyList); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java index e1a5196e5f64f..29d5d78e0ea96 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeResolver.java @@ -51,17 +51,18 @@ public CompletableFuture get(DataFetchingEnvironment environment) { return CompletableFuture.completedFuture(null); } - final String urn = resolvedEntity.getUrn(); final Object javaObject = _entityProvider.apply(environment); if (isOnlySelectingIdentityFields(environment)) { return CompletableFuture.completedFuture(javaObject); } - final com.linkedin.datahub.graphql.types.EntityType filteredEntity = Iterables.getOnlyElement(_entityTypes.stream() + final com.linkedin.datahub.graphql.types.EntityType filteredEntity = Iterables.getOnlyElement(_entityTypes.stream() .filter(entity -> javaObject.getClass().isAssignableFrom(entity.objectClass())) .collect(Collectors.toList())); - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(filteredEntity.name()); - return loader.load(urn); + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(filteredEntity.name()); + final Object key = filteredEntity.getKeyProvider().apply(resolvedEntity); + + return loader.load(key); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java index b9d5d406faa12..d4846a56f95de 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java @@ -23,20 +23,20 @@ public class LoadableTypeBatchResolver implements DataFetcher>> { private final LoadableType _loadableType; - private final Function> _urnProvider; + private final Function> _keyProvider; - public LoadableTypeBatchResolver(final LoadableType loadableType, final Function> urnProvider) { + public LoadableTypeBatchResolver(final LoadableType loadableType, final Function> keyProvider) { _loadableType = loadableType; - _urnProvider = urnProvider; + _keyProvider = keyProvider; } @Override public CompletableFuture> get(DataFetchingEnvironment environment) { - final List urns = _urnProvider.apply(environment); - if (urns == null) { + final List keys = _keyProvider.apply(environment); + if (keys == null) { return null; } - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(_loadableType.name()); - return loader.loadMany(urns); + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(_loadableType.name()); + return loader.loadMany(keys); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java index c260d11663549..b1edc76165084 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java @@ -31,11 +31,11 @@ public LoadableTypeResolver(final LoadableType loadableType, final Functio @Override public CompletableFuture get(DataFetchingEnvironment environment) { - final K urn = _keyProvider.apply(environment); - if (urn == null) { + final K key = _keyProvider.apply(environment); + if (key == null) { return null; } final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(_loadableType.name()); - return loader.load(urn); + return loader.load(key); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java index b6ce3e973a252..a970c813ef172 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java @@ -1,6 +1,8 @@ package com.linkedin.datahub.graphql.types; import com.linkedin.datahub.graphql.generated.Entity; +import java.util.function.Function; + /** * GQL graph type representing a top-level GMS entity (eg. Dataset, User, DataPlatform, Chart, etc.). @@ -14,4 +16,6 @@ public interface EntityType extends LoadableType { */ com.linkedin.datahub.graphql.generated.EntityType type(); + Function getKeyProvider(); + } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java index 5fd3a02f6c436..3493afdd8bd84 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java @@ -4,6 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Assertion; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; @@ -15,6 +16,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -37,6 +39,11 @@ public EntityType type() { return EntityType.ASSERTION; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return Assertion.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index 17c09559161ca..6ac64588fbf5d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.Chart; import com.linkedin.datahub.graphql.generated.ChartUpdateInput; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -46,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -88,6 +90,11 @@ public EntityType type() { return EntityType.CHART; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return Chart.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java index f985183ad06ec..d84615ec923d7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java @@ -4,6 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Container; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.container.mappers.ContainerMapper; import com.linkedin.entity.EntityResponse; @@ -16,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -47,6 +49,11 @@ public EntityType type() { return EntityType.CONTAINER; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return Container.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java index e09f724cf8bf5..e0feb3c903c76 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java @@ -13,6 +13,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.CorpGroup; import com.linkedin.datahub.graphql.generated.CorpGroupUpdateInput; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -36,6 +37,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -64,6 +66,11 @@ public EntityType type() { return EntityType.CORP_GROUP; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public List> batchLoad(final List urns, final QueryContext context) { try { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java index ec1f3a4889047..06c5245d4b657 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java @@ -14,6 +14,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.CorpUserUpdateInput; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -39,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -64,6 +66,11 @@ public EntityType type() { return EntityType.CORP_USER; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public List> batchLoad(final List urns, final QueryContext context) { try { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index 5d4faa8dec6bf..9e2c77b347af2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.Dashboard; import com.linkedin.datahub.graphql.generated.DashboardUpdateInput; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -89,6 +91,11 @@ public EntityType type() { return EntityType.DASHBOARD; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return Dashboard.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index e14d3d206cb60..94a3fe838ac74 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.DataFlow; import com.linkedin.datahub.graphql.generated.DataFlowUpdateInput; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -46,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -81,6 +83,11 @@ public EntityType type() { return EntityType.DATA_FLOW; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return DataFlow.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index 4ba73fcc4d987..d92ec21bd2d87 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.DataJob; import com.linkedin.datahub.graphql.generated.DataJobUpdateInput; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -46,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -82,6 +84,11 @@ public EntityType type() { return EntityType.DATA_JOB; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return DataJob.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java index c601602d6ab33..57a035d136645 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java @@ -4,6 +4,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.DataPlatform; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.types.EntityType; import com.linkedin.datahub.graphql.types.dataplatform.mappers.DataPlatformMapper; import com.linkedin.entity.EntityResponse; @@ -13,6 +14,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import static com.linkedin.metadata.Constants.*; @@ -62,4 +64,9 @@ public List> batchLoad(final List urns, public com.linkedin.datahub.graphql.generated.EntityType type() { return com.linkedin.datahub.graphql.generated.EntityType.DATA_PLATFORM; } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index facacb19c7551..8afb12a51865a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -16,6 +16,7 @@ import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.DatasetUpdateInput; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -46,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -100,6 +102,11 @@ public EntityType type() { return EntityType.DATASET; } + @Override + public Function> getKeyProvider() { + return entity -> new Pair<>(entity.getUrn(), null); + } + @Override public List> batchLoad(@Nonnull final List> versionedUrns, @Nonnull final QueryContext context) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java index 6df558ada3722..b7802a1f234b7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java @@ -4,6 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Domain; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; @@ -15,6 +16,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -38,6 +40,11 @@ public EntityType type() { return EntityType.DOMAIN; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return Domain.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java index 2e9ae8883bee1..2d9ff4d68d0cd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java @@ -8,6 +8,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.GlossaryTerm; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -70,6 +72,11 @@ public EntityType type() { return EntityType.GLOSSARY_TERM; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public List> batchLoad(final List urns, final QueryContext context) { final List glossaryTermUrns = urns.stream() diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java index 685f2aa084f98..87566228dd6d4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java @@ -8,6 +8,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.MLFeatureTable; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -53,6 +55,11 @@ public EntityType type() { return EntityType.MLFEATURE_TABLE; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return MLFeatureTable.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java index 577697aa66860..811b5e4ead721 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureType.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.MLFeature; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -44,6 +46,11 @@ public EntityType type() { return EntityType.MLFEATURE; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return MLFeature.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java index aa890604374eb..3c6a6c1db86e1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java @@ -8,6 +8,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.MLModelGroup; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -53,6 +55,11 @@ public EntityType type() { return EntityType.MLMODEL_GROUP; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return MLModelGroup.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java index 375101dcea3d7..ce9f460dc02b5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java @@ -8,6 +8,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.MLModel; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -52,6 +54,11 @@ public EntityType type() { return EntityType.MLMODEL; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return MLModel.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java index ded6f33179e3e..068bda3b0c064 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLPrimaryKeyType.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.MLPrimaryKey; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -44,6 +46,11 @@ public EntityType type() { return EntityType.MLPRIMARY_KEY; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return MLPrimaryKey.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java index 0735fab12f76b..ba715d990e8b0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java @@ -15,6 +15,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -123,6 +125,11 @@ public EntityType type() { return EntityType.NOTEBOOK; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class objectClass() { return Notebook.class; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java index 369f5911a1c9f..41ae275f4242b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java @@ -10,6 +10,7 @@ import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -36,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -64,6 +66,11 @@ public EntityType type() { return EntityType.TAG; } + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + @Override public Class inputClass() { return TagUpdateInput.class; diff --git a/metadata-ingestion/src/datahub/cli/timeline_cli.py b/metadata-ingestion/src/datahub/cli/timeline_cli.py index 542d962dc4dda..58cf2c7649c08 100644 --- a/metadata-ingestion/src/datahub/cli/timeline_cli.py +++ b/metadata-ingestion/src/datahub/cli/timeline_cli.py @@ -204,8 +204,8 @@ def timeline( if change_txn["changeEvents"] is not None: for change_event in change_txn["changeEvents"]: element_string = ( - f"({pretty_id(change_event.get('elementId'))})" - if change_event.get("elementId") + f"({pretty_id(change_event.get('modifier'))})" + if change_event.get("modifier") else "" ) event_change_color: str = ( @@ -213,9 +213,9 @@ def timeline( if change_event.get("semVerChange") == "MINOR" else "red" ) - target_string = pretty_id(change_event.get("target") or "") + target_string = pretty_id(change_event.get("entityUrn") or "") print( - f"\t{colored(change_event['changeType'],event_change_color)} {change_event.get('category')} {target_string} {element_string}: {change_event['description']}" + f"\t{colored(change_event['operation'],event_change_color)} {change_event.get('category')} {target_string} {element_string}: {change_event['description']}" ) else: click.echo( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index df66ff0991d1f..51e579aa7ffb9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -182,7 +182,6 @@ public Map getEntitiesV2( * Retrieves the aspects for the given set of urns and versions as dynamic aspect objects * (Without having to define union objects) * - * @param entityName name of the entity to fetch * @param versionedUrnStrs set of urns to fetch with versions of aspects specified in a specialized string * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link Entity} object @@ -212,7 +211,6 @@ public abstract Map> getLatestEnvelopedAspects( /** * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects * - * @param entityName name of the entity to fetch * @param versionedUrnStrs set of urns to fetch with versions of aspects specified in a specialized string * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link EnvelopedAspect} object From 8148210f5fd4c96267c8da80c61e3ab918d3dc27 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 21 Apr 2022 23:04:36 -0500 Subject: [PATCH 05/21] feat(versionedDataset): remove unused import --- .../linkedin/metadata/timeline/ebean/EbeanTimelineService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java index 3ab2648fa2e7f..3230765de0f27 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java @@ -33,7 +33,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; From 341586618a1d29eed769929bc892a4fcd9c5afc3 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Fri, 22 Apr 2022 18:39:34 -0500 Subject: [PATCH 06/21] feat(graphql): versioned dataset query - undeprecate schema for non-versioned usage --- .../datahub/graphql/types/dataset/DatasetType.java | 3 ++- .../graphql/types/dataset/mappers/SchemaMapper.java | 7 ++++++- .../src/main/resources/entity.graphql | 12 +++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 8afb12a51865a..3c6fae5a4b10a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -75,7 +75,8 @@ public class DatasetType implements SearchableEntityType FACET_FIELDS = ImmutableSet.of("origin", "platform"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java index 05db591f42e63..7e068d5d414e0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java @@ -18,7 +18,7 @@ public static Schema map(@Nonnull final SchemaMetadata metadata) { @Override public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) { final Schema result = new Schema(); - if (input.hasDataset()) { + if (input.getDataset() != null) { result.setDatasetUrn(input.getDataset().toString()); } result.setName(input.getSchemaName()); @@ -29,6 +29,11 @@ public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) { result.setPrimaryKeys(input.getPrimaryKeys()); result.setFields(input.getFields().stream().map(SchemaFieldMapper::map).collect(Collectors.toList())); result.setPlatformSchema(PlatformSchemaMapper.map(input.getPlatformSchema())); + if (input.getForeignKeys() != null) { + result.setForeignKeys(input.getForeignKeys().stream() + .map(ForeignKeyConstraintMapper::map) + .collect(Collectors.toList())); + } return result; } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index b10c4bb4acc5f..e78f3f25fc91a 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -800,7 +800,7 @@ type Dataset implements EntityWithRelationships & Entity { """ Schema metadata of the dataset """ - schema: Schema @deprecated(reason: "Use `schemaMetadata`") + schema: Schema """ Deprecated, use properties field instead @@ -1733,6 +1733,16 @@ type Schema { Client provided list of fields that define primary keys to access record """ primaryKeys: [String!] + + """ + Client provided list of foreign key constraints + """ + foreignKeys: [ForeignKeyConstraint] + + """ + The time at which the schema metadata information was created + """ + createdAt: Long } """ From cacce4921d1bd1219b0f7a02e3ab28f8fa27d854 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Fri, 22 Apr 2022 19:32:49 -0500 Subject: [PATCH 07/21] feat(graphql): versioned dataset query - fix versionstamp creation bug --- .../linkedin/metadata/timeline/ebean/EbeanTimelineService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java index 3230765de0f27..d0c8d10eb25a4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java @@ -282,6 +282,7 @@ private Map> constructTimestampVersionCache(Map Date: Mon, 25 Apr 2022 17:00:01 -0500 Subject: [PATCH 08/21] feat(graphql): versioned dataset query - revert timeline cli change --- metadata-ingestion/src/datahub/cli/timeline_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/timeline_cli.py b/metadata-ingestion/src/datahub/cli/timeline_cli.py index 58cf2c7649c08..542d962dc4dda 100644 --- a/metadata-ingestion/src/datahub/cli/timeline_cli.py +++ b/metadata-ingestion/src/datahub/cli/timeline_cli.py @@ -204,8 +204,8 @@ def timeline( if change_txn["changeEvents"] is not None: for change_event in change_txn["changeEvents"]: element_string = ( - f"({pretty_id(change_event.get('modifier'))})" - if change_event.get("modifier") + f"({pretty_id(change_event.get('elementId'))})" + if change_event.get("elementId") else "" ) event_change_color: str = ( @@ -213,9 +213,9 @@ def timeline( if change_event.get("semVerChange") == "MINOR" else "red" ) - target_string = pretty_id(change_event.get("entityUrn") or "") + target_string = pretty_id(change_event.get("target") or "") print( - f"\t{colored(change_event['operation'],event_change_color)} {change_event.get('category')} {target_string} {element_string}: {change_event['description']}" + f"\t{colored(change_event['changeType'],event_change_color)} {change_event.get('category')} {target_string} {element_string}: {change_event['description']}" ) else: click.echo( From 563e4f97868a8b6795922b654ca690798c222d54 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 13:11:40 -0500 Subject: [PATCH 09/21] feat(graphql): versioned dataset query - handle partial versionStamps properly --- .../metadata/entity/ebean/EbeanEntityService.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 727a9c0f9d843..3f19a32285604 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -230,14 +230,26 @@ public Map> getVersionedEnvelopedAspects( .collect(Collectors.toMap(Pair::getFirst, pair -> EntityUtils.convertVersionStamp(pair.getSecond()))); + // Cover full/partial versionStamp final Set dbKeys = urnAspectVersionMap.entrySet().stream() + .filter(entry -> !entry.getValue().isEmpty()) .map(entry -> aspectNames.stream() + .filter(aspectName -> entry.getValue().containsKey(aspectName)) .map(aspectName -> new EbeanAspectV2.PrimaryKey(entry.getKey(), aspectName, - entry.getValue().getOrDefault(aspectName, 0L))) + entry.getValue().get(aspectName))) .collect(Collectors.toList())) .flatMap(List::stream) .collect(Collectors.toSet()); + // Cover empty versionStamp + dbKeys.addAll(urnAspectVersionMap.entrySet().stream() + .filter(entry -> entry.getValue().isEmpty()) + .map(entry -> aspectNames.stream() + .map(aspectName -> new EbeanAspectV2.PrimaryKey(entry.getKey(), aspectName, 0L)) + .collect(Collectors.toList())) + .flatMap(List::stream) + .collect(Collectors.toSet())); + return getCorrespondingAspects(dbKeys, versionedUrnStrs.stream() .map(Pair::getFirst) .map(UrnUtils::getUrn).collect(Collectors.toSet())); From 9c91147cbd36bdea888214a9e5067eb202b137d2 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 15:05:38 -0500 Subject: [PATCH 10/21] feat(graphql): versioned dataset query - fix import --- .../com/linkedin/metadata/entity/ebean/EbeanEntityService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 6bfb42ce1209f..ed5f196a80903 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -9,6 +9,7 @@ import com.linkedin.common.Status; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.JacksonDataTemplateCodec; From 23cb9052b18c3decd131424edf9857cec55b5b4c Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 15:47:57 -0500 Subject: [PATCH 11/21] feat(graphql): versioned dataset query - update cassandra service to implement interface methods --- .../cassandra/CassandraEntityService.java | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java index fdb3a1efb4d11..5995687490e75 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java @@ -142,7 +142,7 @@ public Map> getLatestEnvelopedAspects( // TODO: entityName is unused, can we remove this as a param? @Nonnull String entityName, @Nonnull Set urns, - @Nonnull Set aspectNames) throws URISyntaxException { + @Nonnull Set aspectNames) { final Set dbKeys = urns.stream() .map(urn -> aspectNames.stream() @@ -151,6 +151,10 @@ public Map> getLatestEnvelopedAspects( .flatMap(List::stream) .collect(Collectors.toSet()); + return getCorrespondingAspects(dbKeys, urns); + } + + private Map> getCorrespondingAspects(Set dbKeys, Set urns) { final Map envelopedAspectMap = getEnvelopedAspects(dbKeys); // Group result by Urn @@ -174,6 +178,39 @@ public Map> getLatestEnvelopedAspects( return result; } + @Override + public Map> getVersionedEnvelopedAspects( + @Nonnull Set> versionedUrnStrs, @Nonnull Set aspectNames) throws URISyntaxException { + + Map> urnAspectVersionMap = versionedUrnStrs.stream() + .collect(Collectors.toMap(Pair::getFirst, + pair -> EntityUtils.convertVersionStamp(pair.getSecond()))); + + // Cover full/partial versionStamp + final Set dbKeys = urnAspectVersionMap.entrySet().stream() + .filter(entry -> !entry.getValue().isEmpty()) + .map(entry -> aspectNames.stream() + .filter(aspectName -> entry.getValue().containsKey(aspectName)) + .map(aspectName -> new CassandraAspect.PrimaryKey(entry.getKey(), aspectName, + entry.getValue().get(aspectName))) + .collect(Collectors.toList())) + .flatMap(List::stream) + .collect(Collectors.toSet()); + + // Cover empty versionStamp + dbKeys.addAll(urnAspectVersionMap.entrySet().stream() + .filter(entry -> entry.getValue().isEmpty()) + .map(entry -> aspectNames.stream() + .map(aspectName -> new CassandraAspect.PrimaryKey(entry.getKey(), aspectName, 0L)) + .collect(Collectors.toList())) + .flatMap(List::stream) + .collect(Collectors.toSet())); + + return getCorrespondingAspects(dbKeys, versionedUrnStrs.stream() + .map(Pair::getFirst) + .map(UrnUtils::getUrn).collect(Collectors.toSet())); + } + @Override public EnvelopedAspect getEnvelopedAspect( // TODO: entityName is only used for a debug statement, can we remove this as a param? @@ -616,7 +653,7 @@ private long calculateVersionNumber(@Nonnull final Urn urn, @Nonnull final Strin return version; } - private Map getEnvelopedAspects(final Set dbKeys) throws URISyntaxException { + private Map getEnvelopedAspects(final Set dbKeys) { final Map result = new HashMap<>(); final Map dbEntries = _aspectDao.batchGet(dbKeys); @@ -637,7 +674,7 @@ private Map getEnvelopedAspects(fin envelopedAspect.setVersion(currAspectEntry.getVersion()); envelopedAspect.setValue(aspect); envelopedAspect.setCreated(new AuditStamp() - .setActor(Urn.createFromString(currAspectEntry.getCreatedBy())) + .setActor(UrnUtils.getUrn(currAspectEntry.getCreatedBy())) .setTime(currAspectEntry.getCreatedOn().getTime()) ); result.put(currKey, envelopedAspect); @@ -645,7 +682,7 @@ private Map getEnvelopedAspects(fin return result; } - private EnvelopedAspect getKeyEnvelopedAspect(final Urn urn) throws URISyntaxException { + private EnvelopedAspect getKeyEnvelopedAspect(final Urn urn) { final EntitySpec spec = getEntityRegistry().getEntitySpec(PegasusUtils.urnToEntityName(urn)); final AspectSpec keySpec = spec.getKeyAspectSpec(); final RecordDataSchema keySchema = keySpec.getPegasusSchema(); @@ -657,7 +694,7 @@ private EnvelopedAspect getKeyEnvelopedAspect(final Urn urn) throws URISyntaxExc envelopedAspect.setVersion(ASPECT_LATEST_VERSION); envelopedAspect.setValue(aspect); envelopedAspect.setCreated( - new AuditStamp().setActor(Urn.createFromString(SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); + new AuditStamp().setActor(UrnUtils.getUrn(SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); return envelopedAspect; } From 2403710d948256c0c2954d4d7a7adcc244d71e8a Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 20:57:55 -0500 Subject: [PATCH 12/21] feat(graphql): versioned dataset query - review comments --- .../datahub/graphql/GmsGraphQLEngine.java | 10 +++++++--- .../load/LoadableTypeBatchResolver.java | 1 + .../resolvers/load/LoadableTypeResolver.java | 1 + .../graphql/types/BrowsableEntityType.java | 1 + .../datahub/graphql/types/EntityType.java | 1 + .../datahub/graphql/types/LoadableType.java | 1 + .../graphql/types/aspect/AspectType.java | 2 +- .../graphql/types/dataset/DatasetType.java | 15 ++++++++------- .../metadata/entity/EntityService.java | 11 ++++++----- .../cassandra/CassandraEntityService.java | 13 +++++++------ .../entity/ebean/EbeanEntityService.java | 13 +++++++------ .../com/linkedin/common/VersionedUrn.pdl | 18 ++++++++++++++++++ .../linkedin/entity/client/EntityClient.java | 3 ++- .../entity/client/JavaEntityClient.java | 3 ++- .../entity/client/RestliEntityClient.java | 4 +++- .../entity/EntityVersionedV2Resource.java | 12 +++++++++++- 16 files changed, 77 insertions(+), 32 deletions(-) create mode 100644 metadata-models/src/main/pegasus/com/linkedin/common/VersionedUrn.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index da2b5628fce7c..c89a612bda8cf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -4,6 +4,8 @@ import com.datahub.authentication.token.TokenService; import com.datahub.authorization.AuthorizationConfiguration; import com.google.common.collect.ImmutableList; +import com.linkedin.common.VersionedUrn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.analytics.resolver.AnalyticsChartTypeResolver; import com.linkedin.datahub.graphql.analytics.resolver.GetChartsResolver; import com.linkedin.datahub.graphql.analytics.resolver.GetHighlightsResolver; @@ -479,8 +481,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("browse", new BrowseResolver(browsableTypes)) .dataFetcher("browsePaths", new BrowsePathsResolver(browsableTypes)) .dataFetcher("dataset", getResolver(datasetType, - (env) -> new Pair<>(env.getArgument(URN_FIELD_NAME), - env.getArgument(VERSION_STAMP_FIELD_NAME)))) + (env) -> new VersionedUrn().setUrn(UrnUtils.getUrn(env.getArgument(URN_FIELD_NAME))) + .setVersionStamp(env.getArgument(VERSION_STAMP_FIELD_NAME)))) .dataFetcher("notebook", getResolver(notebookType)) .dataFetcher("corpUser", getResolver(corpUserType)) .dataFetcher("corpGroup", getResolver(corpGroupType)) @@ -687,7 +689,9 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { ) .type("ForeignKeyConstraint", typeWiring -> typeWiring .dataFetcher("foreignDataset", new LoadableTypeResolver<>(datasetType, - (env) -> new Pair<>(((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn(), null))) + (env) -> new VersionedUrn().setUrn( + UrnUtils.getUrn( + ((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn())))) ) .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring .dataFetcher("author", new LoadableTypeResolver<>(corpUserType, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java index d4846a56f95de..fe8ebc8244a13 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java @@ -19,6 +19,7 @@ * for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()} * * @param the generated GraphQL POJO corresponding to the resolved type. + * @param the key type for the dataloader */ public class LoadableTypeBatchResolver implements DataFetcher>> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java index b1edc76165084..5acf9471f21e3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java @@ -18,6 +18,7 @@ * for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()} * * @param the generated GraphQL POJO corresponding to the resolved type. + * @param the key type for the dataloader */ public class LoadableTypeResolver implements DataFetcher> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java index 3f9e74d628882..ae31f21a17c41 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java @@ -14,6 +14,7 @@ * Extension of {@link EntityType} containing methods required for 'browse' functionality. * * @param : The GraphQL object type corresponding to the entity, must extend the `Entity` interface. + * @param the key type for the dataloader */ public interface BrowsableEntityType extends EntityType { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java index a970c813ef172..063e963d57fdf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java @@ -8,6 +8,7 @@ * GQL graph type representing a top-level GMS entity (eg. Dataset, User, DataPlatform, Chart, etc.). * * @param : The GraphQL object type corresponding to the entity, must be of type {@link Entity} + * @param the key type for the dataloader */ public interface EntityType extends LoadableType { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java index 949a582978da7..d2231cfa46ff1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java @@ -11,6 +11,7 @@ * GQL graph type that can be loaded from a downstream service by primary key. * * @param : The GraphQL object type corresponding to the type. + * @param the key type for the dataloader */ public interface LoadableType { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java index 56ecdba9a68ab..50762d08ab55a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -16,7 +16,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; - +@Deprecated public class AspectType implements LoadableType { private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 3c6fae5a4b10a..b3ef66876db68 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -56,7 +57,7 @@ import static com.linkedin.metadata.Constants.*; -public class DatasetType implements SearchableEntityType>, BrowsableEntityType>, +public class DatasetType implements SearchableEntityType, BrowsableEntityType, MutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( @@ -104,12 +105,12 @@ public EntityType type() { } @Override - public Function> getKeyProvider() { - return entity -> new Pair<>(entity.getUrn(), null); + public Function getKeyProvider() { + return entity -> new VersionedUrn().setUrn(UrnUtils.getUrn(entity.getUrn())); } @Override - public List> batchLoad(@Nonnull final List> versionedUrns, + public List> batchLoad(@Nonnull final List versionedUrns, @Nonnull final QueryContext context) { try { final Map datasetMap = @@ -120,8 +121,8 @@ public List> batchLoad(@Nonnull final List gmsResults = new ArrayList<>(); - for (Pair pair : versionedUrns) { - gmsResults.add(datasetMap.getOrDefault(UrnUtils.getUrn(pair.getFirst()), null)); + for (VersionedUrn versionedUrn : versionedUrns) { + gmsResults.add(datasetMap.getOrDefault(versionedUrn.getUrn(), null)); } return gmsResults.stream() .map(gmsDataset -> gmsDataset == null ? null : DataFetcherResult.newResult() @@ -192,7 +193,7 @@ public Dataset update(@Nonnull String urn, @Nonnull DatasetUpdateInput input, @N throw new RuntimeException(String.format("Failed to write entity with urn %s", urn), e); } - return load(new Pair<>(urn, null), context).getData(); + return load(new VersionedUrn().setUrn(UrnUtils.getUrn(urn)), context).getData(); } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index ca0b9f2594f6e..bb2aaa0e248f9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -9,6 +9,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.BrowsePaths; import com.linkedin.common.Status; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.RecordDataSchema; @@ -186,14 +187,14 @@ public Map getEntitiesV2( * Retrieves the aspects for the given set of urns and versions as dynamic aspect objects * (Without having to define union objects) * - * @param versionedUrnStrs set of urns to fetch with versions of aspects specified in a specialized string + * @param versionedUrns set of urns to fetch with versions of aspects specified in a specialized string * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link Entity} object */ public Map getEntitiesVersionedV2( - @Nonnull final Set> versionedUrnStrs, + @Nonnull final Set versionedUrns, @Nonnull final Set aspectNames) throws URISyntaxException { - return getVersionedEnvelopedAspects(versionedUrnStrs, aspectNames) + return getVersionedEnvelopedAspects(versionedUrns, aspectNames) .entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> toEntityResponse(entry.getKey(), entry.getValue()))); @@ -215,12 +216,12 @@ public abstract Map> getLatestEnvelopedAspects( /** * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects * - * @param versionedUrnStrs set of urns to fetch with versions of aspects specified in a specialized string + * @param versionedUrns set of urns to fetch with versions of aspects specified in a specialized string * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link EnvelopedAspect} object */ public abstract Map> getVersionedEnvelopedAspects( - @Nonnull final Set> versionedUrnStrs, + @Nonnull final Set versionedUrns, @Nonnull final Set aspectNames) throws URISyntaxException; /** diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java index 5995687490e75..04356925f1fdb 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java @@ -8,6 +8,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.Status; import com.linkedin.common.UrnArray; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.RecordDataSchema; @@ -180,11 +181,11 @@ private Map> getCorrespondingAspects(Set> getVersionedEnvelopedAspects( - @Nonnull Set> versionedUrnStrs, @Nonnull Set aspectNames) throws URISyntaxException { + @Nonnull Set versionedUrns, @Nonnull Set aspectNames) throws URISyntaxException { - Map> urnAspectVersionMap = versionedUrnStrs.stream() - .collect(Collectors.toMap(Pair::getFirst, - pair -> EntityUtils.convertVersionStamp(pair.getSecond()))); + Map> urnAspectVersionMap = versionedUrns.stream() + .collect(Collectors.toMap(versionedUrn -> versionedUrn.getUrn().toString(), + versionedUrn -> EntityUtils.convertVersionStamp(versionedUrn.getVersionStamp()))); // Cover full/partial versionStamp final Set dbKeys = urnAspectVersionMap.entrySet().stream() @@ -206,8 +207,8 @@ public Map> getVersionedEnvelopedAspects( .flatMap(List::stream) .collect(Collectors.toSet())); - return getCorrespondingAspects(dbKeys, versionedUrnStrs.stream() - .map(Pair::getFirst) + return getCorrespondingAspects(dbKeys, versionedUrns.stream() + .map(versionedUrn -> versionedUrn.getUrn().toString()) .map(UrnUtils::getUrn).collect(Collectors.toSet())); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index ed5f196a80903..28d2e56c2a32b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -8,6 +8,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.Status; import com.linkedin.common.UrnArray; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.RecordDataSchema; @@ -206,12 +207,12 @@ public Map> getLatestEnvelopedAspects( @Override public Map> getVersionedEnvelopedAspects( - @Nonnull Set> versionedUrnStrs, + @Nonnull Set versionedUrns, @Nonnull Set aspectNames) throws URISyntaxException { - Map> urnAspectVersionMap = versionedUrnStrs.stream() - .collect(Collectors.toMap(Pair::getFirst, - pair -> EntityUtils.convertVersionStamp(pair.getSecond()))); + Map> urnAspectVersionMap = versionedUrns.stream() + .collect(Collectors.toMap(versionedUrn -> versionedUrn.getUrn().toString(), + versionedUrn -> EntityUtils.convertVersionStamp(versionedUrn.getVersionStamp()))); // Cover full/partial versionStamp final Set dbKeys = urnAspectVersionMap.entrySet().stream() @@ -233,8 +234,8 @@ public Map> getVersionedEnvelopedAspects( .flatMap(List::stream) .collect(Collectors.toSet())); - return getCorrespondingAspects(dbKeys, versionedUrnStrs.stream() - .map(Pair::getFirst) + return getCorrespondingAspects(dbKeys, versionedUrns.stream() + .map(versionedUrn -> versionedUrn.getUrn().toString()) .map(UrnUtils::getUrn).collect(Collectors.toSet())); } diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/VersionedUrn.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/VersionedUrn.pdl new file mode 100644 index 0000000000000..10c8bc2944316 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/VersionedUrn.pdl @@ -0,0 +1,18 @@ +namespace com.linkedin.common + +/** + * An urn combined with a versionStamp, versionStamp can be empty indicating latest for all aspects. Represents a state in time + * of an entity + */ +record VersionedUrn { + + /** + * Urn indicating which entity this represents + */ + urn: Urn + + /** + * String indicating the stamped version of aspects present at a point in time + */ + versionStamp: optional string +} \ No newline at end of file diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java index d49c2bffb9774..e27e3e293e52f 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -1,6 +1,7 @@ package com.linkedin.entity.client; import com.datahub.authentication.Authentication; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; @@ -58,7 +59,7 @@ public Map batchGetV2( @Nonnull Map batchGetVersionedV2( @Nonnull String entityName, - @Nonnull final Set> versionedUrns, + @Nonnull final Set versionedUrns, @Nullable final Set aspectNames, @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException; diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java index e41a4620e51e7..01d505f551042 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java @@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.aspect.GetTimeseriesAspectValuesResponse; import com.linkedin.common.AuditStamp; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; @@ -102,7 +103,7 @@ public Map batchGetV2( @Nonnull public Map batchGetVersionedV2( @Nonnull String entityName, - @Nonnull final Set> versionedUrns, + @Nonnull final Set versionedUrns, @Nullable final Set aspectNames, @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException { final Set projectedAspects = aspectNames == null diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index 1c012e699f1c7..b619010bd0c9f 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -2,6 +2,7 @@ import com.datahub.authentication.Authentication; import com.datahub.util.RecordUtils; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.client.BaseClient; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; @@ -189,7 +190,7 @@ public Map batchGetV2(@Nonnull String entityName, @Nonnull @Nonnull public Map batchGetVersionedV2( @Nonnull String entityName, - @Nonnull final Set> versionedUrns, + @Nonnull final Set versionedUrns, @Nullable final Set aspectNames, @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException { @@ -197,6 +198,7 @@ public Map batchGetVersionedV2( .aspectsParam(aspectNames) .entityTypeParam(entityName) .ids(versionedUrns.stream() + .map(versionedUrn -> new Pair<>(versionedUrn.getUrn(), versionedUrn.getVersionStamp())) .map(Pair::toString) .map(string -> new PairCoercer().coerceOutput(string)).collect( Collectors.toSet())); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java index 606fb6b438d4a..7c6250477c4af 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java @@ -1,7 +1,9 @@ package com.linkedin.metadata.resources.entity; import com.codahale.metrics.MetricRegistry; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.entity.EntityResponse; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.restli.RestliUtil; @@ -18,6 +20,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; @@ -55,7 +58,14 @@ public Task> batchGetVersioned( final Set projectedAspects = aspectNames == null ? getAllAspectNames(_entityService, entityType) : new HashSet<>(Arrays.asList(aspectNames)); try { - return _entityService.getEntitiesVersionedV2(versionedUrnStrs, projectedAspects); + return _entityService.getEntitiesVersionedV2(versionedUrnStrs.stream() + .map(pair -> { + VersionedUrn versionedUrn = new VersionedUrn().setUrn(UrnUtils.getUrn(pair.getFirst())); + if (pair.getSecond() != null) { + versionedUrn.setVersionStamp(pair.getSecond()); + } + return versionedUrn; + }).collect(Collectors.toSet()), projectedAspects); } catch (Exception e) { throw new RuntimeException( String.format("Failed to batch get versioned entities: %s, projectedAspects: %s", versionedUrnStrs, projectedAspects), From edc2487f42805d75232693bc7d2ba7a88a8c5c7e Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 21:25:14 -0500 Subject: [PATCH 13/21] feat(graphql): dataset versioned query - add test --- .../entity/EntityServiceTestBase.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTestBase.java index 856b6371a5cf6..56d1663a1b5b4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTestBase.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.Status; +import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.ByteString; @@ -311,6 +312,79 @@ public void testIngestGetEntitiesV2() throws Exception { verifyNoMoreInteractions(_mockProducer); } + @Test + public void testIngestGetEntitiesVersionedV2() throws Exception { + // Test Writing a CorpUser Entity + Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:tester1"); + VersionedUrn versionedUrn1 = new VersionedUrn().setUrn(entityUrn1).setVersionStamp("corpUserInfo:0"); + com.linkedin.entity.Entity writeEntity1 = createCorpUserEntity(entityUrn1, "tester@test.com"); + + Urn entityUrn2 = Urn.createFromString("urn:li:corpuser:tester2"); + VersionedUrn versionedUrn2 = new VersionedUrn().setUrn(entityUrn2); + com.linkedin.entity.Entity writeEntity2 = createCorpUserEntity(entityUrn2, "tester2@test.com"); + + SystemMetadata metadata1 = new SystemMetadata(); + metadata1.setLastObserved(1625792689); + metadata1.setRunId("run-123"); + + SystemMetadata metadata2 = new SystemMetadata(); + metadata2.setLastObserved(1625792690); + metadata2.setRunId("run-123"); + + String aspectName = "corpUserInfo"; + String keyName = "corpUserKey"; + + // 1. Ingest Entities + _entityService.ingestEntities(ImmutableList.of(writeEntity1, writeEntity2), TEST_AUDIT_STAMP, + ImmutableList.of(metadata1, metadata2)); + + // 2. Retrieve Entities + Map readEntities = + _entityService.getEntitiesVersionedV2(ImmutableSet.of(versionedUrn1, versionedUrn2), ImmutableSet.of(aspectName)); + + // 3. Compare Entity Objects + + // Entity 1 + EntityResponse readEntityResponse1 = readEntities.get(entityUrn1); + assertEquals(2, readEntityResponse1.getAspects().size()); // Key + Info aspect. + EnvelopedAspect envelopedAspect1 = readEntityResponse1.getAspects().get(aspectName); + assertEquals(envelopedAspect1.getName(), aspectName); + assertTrue( + DataTemplateUtil.areEqual(writeEntity1.getValue().getCorpUserSnapshot().getAspects().get(0).getCorpUserInfo(), + new CorpUserInfo(envelopedAspect1.getValue().data()))); + CorpUserKey expectedKey1 = new CorpUserKey(); + expectedKey1.setUsername("tester1"); + EnvelopedAspect envelopedKey1 = readEntityResponse1.getAspects().get(keyName); + assertTrue(DataTemplateUtil.areEqual(expectedKey1, new CorpUserKey(envelopedKey1.getValue().data()))); + + // Entity 2 + EntityResponse readEntityResponse2 = readEntities.get(entityUrn2); + assertEquals(2, readEntityResponse2.getAspects().size()); // Key + Info aspect. + EnvelopedAspect envelopedAspect2 = readEntityResponse2.getAspects().get(aspectName); + assertEquals(envelopedAspect2.getName(), aspectName); + assertTrue( + DataTemplateUtil.areEqual(writeEntity2.getValue().getCorpUserSnapshot().getAspects().get(0).getCorpUserInfo(), + new CorpUserInfo(envelopedAspect2.getValue().data()))); + CorpUserKey expectedKey2 = new CorpUserKey(); + expectedKey2.setUsername("tester2"); + EnvelopedAspect envelopedKey2 = readEntityResponse2.getAspects().get(keyName); + assertTrue(DataTemplateUtil.areEqual(expectedKey2, new CorpUserKey(envelopedKey2.getValue().data()))); + + verify(_mockProducer, times(2)).produceMetadataAuditEvent(Mockito.eq(entityUrn1), Mockito.eq(null), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.eq(MetadataAuditOperation.UPDATE)); + + verify(_mockProducer, times(2)).produceMetadataAuditEvent(Mockito.eq(entityUrn2), Mockito.eq(null), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.eq(MetadataAuditOperation.UPDATE)); + + verify(_mockProducer, times(2)).produceMetadataChangeLog(Mockito.eq(entityUrn1), + Mockito.any(), Mockito.any()); + + verify(_mockProducer, times(2)).produceMetadataChangeLog(Mockito.eq(entityUrn2), + Mockito.any(), Mockito.any()); + + verifyNoMoreInteractions(_mockProducer); + } + @Test public void testIngestAspectsGetLatestAspects() throws Exception { From 36b08db4392f16360207a8ceff60505207bb05c1 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 21:26:16 -0500 Subject: [PATCH 14/21] checkstyle --- .../main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index c89a612bda8cf..000e8470a5ecf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -168,7 +168,6 @@ import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.version.GitVersion; import com.linkedin.usage.UsageClient; -import com.linkedin.util.Pair; import graphql.execution.DataFetcherResult; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; From 4f923ba94eda25a0c8252ec2754f2470e09eb81f Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 21:46:56 -0500 Subject: [PATCH 15/21] checkstyle --- .../com/linkedin/datahub/graphql/types/dataset/DatasetType.java | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index b3ef66876db68..f248df4273deb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -40,7 +40,6 @@ import com.linkedin.metadata.search.SearchResult; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.util.Pair; import graphql.execution.DataFetcherResult; import java.util.ArrayList; import java.util.Collection; From a7f29f1f295057e9e7e5a4ad60cb7dfe36c52275 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 22:40:22 -0500 Subject: [PATCH 16/21] change comment, force rebuild --- .../graphql/resolvers/load/LoadableTypeBatchResolver.java | 2 +- .../datahub/graphql/resolvers/load/LoadableTypeResolver.java | 2 +- .../com/linkedin/datahub/graphql/types/BrowsableEntityType.java | 2 +- .../java/com/linkedin/datahub/graphql/types/EntityType.java | 2 +- .../java/com/linkedin/datahub/graphql/types/LoadableType.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java index fe8ebc8244a13..02a92544855a3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java @@ -19,7 +19,7 @@ * for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()} * * @param the generated GraphQL POJO corresponding to the resolved type. - * @param the key type for the dataloader + * @param the key type for the DataLoader */ public class LoadableTypeBatchResolver implements DataFetcher>> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java index 5acf9471f21e3..53702f9cafe8b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeResolver.java @@ -18,7 +18,7 @@ * for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()} * * @param the generated GraphQL POJO corresponding to the resolved type. - * @param the key type for the dataloader + * @param the key type for the DataLoader */ public class LoadableTypeResolver implements DataFetcher> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java index ae31f21a17c41..f3de6fd7c1852 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BrowsableEntityType.java @@ -14,7 +14,7 @@ * Extension of {@link EntityType} containing methods required for 'browse' functionality. * * @param : The GraphQL object type corresponding to the entity, must extend the `Entity` interface. - * @param the key type for the dataloader + * @param the key type for the DataLoader */ public interface BrowsableEntityType extends EntityType { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java index 063e963d57fdf..4185288776c06 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/EntityType.java @@ -8,7 +8,7 @@ * GQL graph type representing a top-level GMS entity (eg. Dataset, User, DataPlatform, Chart, etc.). * * @param : The GraphQL object type corresponding to the entity, must be of type {@link Entity} - * @param the key type for the dataloader + * @param the key type for the DataLoader */ public interface EntityType extends LoadableType { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java index d2231cfa46ff1..a21fab09b79c3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java @@ -11,7 +11,7 @@ * GQL graph type that can be loaded from a downstream service by primary key. * * @param : The GraphQL object type corresponding to the type. - * @param the key type for the dataloader + * @param the key type for the DataLoader */ public interface LoadableType { From ddda27ab00f156afc456832c2e0161882d6fcb83 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Wed, 27 Apr 2022 23:39:44 -0500 Subject: [PATCH 17/21] checkstyle --- .../src/main/java/com/linkedin/entity/client/EntityClient.java | 1 - .../main/java/com/linkedin/entity/client/JavaEntityClient.java | 1 - 2 files changed, 2 deletions(-) diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java index e27e3e293e52f..3303e095f44b1 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -23,7 +23,6 @@ import com.linkedin.mxe.PlatformEvent; import com.linkedin.mxe.SystemMetadata; import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.util.Pair; import java.net.URISyntaxException; import java.util.Collection; import java.util.List; diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java index 01d505f551042..dca2fc1b191a6 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java @@ -38,7 +38,6 @@ import com.linkedin.mxe.PlatformEvent; import com.linkedin.mxe.SystemMetadata; import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.util.Pair; import io.opentelemetry.extension.annotations.WithSpan; import java.net.URISyntaxException; import java.time.Clock; From 842736066e7e69641edaa46e4b90175951745ea2 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 28 Apr 2022 15:18:04 -0500 Subject: [PATCH 18/21] feat(graphql): versioned dataset query - review comments: change from Pair to stronger type --- .../java/com/linkedin/util/PairCoercer.java | 24 -------- .../linkedin/util/VersionedUrnCoercer.java | 25 +++++++++ .../com/linkedin/common/urn/VersionedUrn.java | 55 +++++++++++++++++++ .../main/pegasus/com/linkedin/common/Pair.pdl | 5 -- .../common/versioned/VersionedUrn.pdl | 5 ++ ...n.entity.entitiesVersionedV2.restspec.json | 2 +- .../com.linkedin.entity.aspects.snapshot.json | 2 +- ...com.linkedin.entity.entities.snapshot.json | 5 +- ...n.entity.entitiesVersionedV2.snapshot.json | 14 ++--- ...m.linkedin.platform.platform.snapshot.json | 5 +- .../entity/client/RestliEntityClient.java | 19 ++----- .../entity/EntityVersionedV2Resource.java | 15 +++-- 12 files changed, 115 insertions(+), 61 deletions(-) delete mode 100644 li-utils/src/main/java/com/linkedin/util/PairCoercer.java create mode 100644 li-utils/src/main/java/com/linkedin/util/VersionedUrnCoercer.java create mode 100644 li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrn.java delete mode 100644 li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl create mode 100644 li-utils/src/main/pegasus/com/linkedin/common/versioned/VersionedUrn.pdl diff --git a/li-utils/src/main/java/com/linkedin/util/PairCoercer.java b/li-utils/src/main/java/com/linkedin/util/PairCoercer.java deleted file mode 100644 index 2787cce437b71..0000000000000 --- a/li-utils/src/main/java/com/linkedin/util/PairCoercer.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.linkedin.util; - -import com.linkedin.data.template.Custom; -import com.linkedin.data.template.DirectCoercer; -import com.linkedin.data.template.TemplateOutputCastException; - - -public class PairCoercer implements DirectCoercer { - static { - Custom.registerCoercer(new PairCoercer(), Pair.class); - } - - @Override - public Object coerceInput(Pair object) throws ClassCastException { - return object.toString(); - } - - @Override - public Pair coerceOutput(Object object) throws TemplateOutputCastException { - String pairStr = (String) object; - String[] split = pairStr.split(","); - return Pair.of(split[0].substring(1), split[1].substring(0, split[1].length() - 1)); - } -} diff --git a/li-utils/src/main/java/com/linkedin/util/VersionedUrnCoercer.java b/li-utils/src/main/java/com/linkedin/util/VersionedUrnCoercer.java new file mode 100644 index 0000000000000..14949d9c946d9 --- /dev/null +++ b/li-utils/src/main/java/com/linkedin/util/VersionedUrnCoercer.java @@ -0,0 +1,25 @@ +package com.linkedin.util; + +import com.linkedin.common.urn.VersionedUrn; +import com.linkedin.data.template.Custom; +import com.linkedin.data.template.DirectCoercer; +import com.linkedin.data.template.TemplateOutputCastException; + + +public class VersionedUrnCoercer implements DirectCoercer { + static { + Custom.registerCoercer(new VersionedUrnCoercer(), VersionedUrn.class); + } + + @Override + public Object coerceInput(VersionedUrn object) throws ClassCastException { + return object.toString(); + } + + @Override + public VersionedUrn coerceOutput(Object object) throws TemplateOutputCastException { + String pairStr = (String) object; + String[] split = pairStr.split(" , "); + return VersionedUrn.of(split[0].substring(1), split[1].substring(0, split[1].length() - 1)); + } +} diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrn.java new file mode 100644 index 0000000000000..d5b7a7da456a9 --- /dev/null +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrn.java @@ -0,0 +1,55 @@ +package com.linkedin.common.urn; + +public class VersionedUrn { + + private final String _urn; + private final String _versionStamp; + + public VersionedUrn(String urn, String versionStamp) { + _urn = urn; + _versionStamp = versionStamp; + } + + public String getUrn() { + return _urn; + } + + public String getVersionStamp() { + return _versionStamp; + } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object obj) { + if (obj instanceof VersionedUrn) { + VersionedUrn other = (VersionedUrn) obj; + return equals(_urn, other._urn) && equals(_versionStamp, other._versionStamp); + } + return false; + } + + @Override + public int hashCode() { + int h1 = _urn != null ? _urn.hashCode() : 0; + int h2 = _versionStamp != null ? _versionStamp.hashCode() : 0; + return 31 * h1 + h2; + } + + @Override + public String toString() { + return "(" + _urn + " , " + _versionStamp + ")"; + } + + private static boolean equals(Object o1, Object o2) { + if (o1 != null) { + return o1.equals(o2); + } + return o2 == null; + } + + /*convenient method*/ + public static VersionedUrn of(String urn, String versionStamp) { + return new VersionedUrn(urn, versionStamp); + } + +} diff --git a/li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl b/li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl deleted file mode 100644 index 130b1bc2a72bc..0000000000000 --- a/li-utils/src/main/pegasus/com/linkedin/common/Pair.pdl +++ /dev/null @@ -1,5 +0,0 @@ -namespace com.linkedin.common - -@java.class = "com.linkedin.util.Pair" -@java.coercerClass = "com.linkedin.util.PairCoercer" -typeref Pair = string \ No newline at end of file diff --git a/li-utils/src/main/pegasus/com/linkedin/common/versioned/VersionedUrn.pdl b/li-utils/src/main/pegasus/com/linkedin/common/versioned/VersionedUrn.pdl new file mode 100644 index 0000000000000..2d49de6a45fb7 --- /dev/null +++ b/li-utils/src/main/pegasus/com/linkedin/common/versioned/VersionedUrn.pdl @@ -0,0 +1,5 @@ +namespace com.linkedin.common.versioned + +@java.class = "com.linkedin.common.urn.VersionedUrn" +@java.coercerClass = "com.linkedin.util.VersionedUrnCoercer" +typeref VersionedUrn = string \ No newline at end of file diff --git a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesVersionedV2.restspec.json b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesVersionedV2.restspec.json index 34934c53cd042..579f1d7c7dddc 100644 --- a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesVersionedV2.restspec.json +++ b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesVersionedV2.restspec.json @@ -7,7 +7,7 @@ "collection" : { "identifier" : { "name" : "entitiesVersionedV2Id", - "type" : "com.linkedin.common.Pair" + "type" : "com.linkedin.common.versioned.VersionedUrn" }, "supports" : [ "batch_get" ], "methods" : [ { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 1808381bf5324..362c522ea8331 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -3433,7 +3433,7 @@ }, { "name" : "aspectName", "type" : "string", - "doc" : "Aspect of the entity being written to\nNot filling this out implies that the writer wants to affect the entire entity\nNote: This is only valid for CREATE and DELETE operations.\n", + "doc" : "Aspect of the entity being written to\nNot filling this out implies that the writer wants to affect the entire entity\nNote: This is only valid for CREATE, UPSERT, and DELETE operations.\n", "optional" : true }, { "name" : "aspect", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 20bba901b33fa..39ba59fec50a5 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -4570,7 +4570,10 @@ "fields" : [ { "name" : "displayName", "type" : "string", - "doc" : "Display name of the Policy" + "doc" : "Display name of the Policy", + "Searchable" : { + "fieldType" : "KEYWORD" + } }, { "name" : "description", "type" : "string", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json index 97bcff084be3f..6e187fba11856 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json @@ -30,16 +30,16 @@ "doc" : "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.", "optional" : true } ] - }, { + }, "com.linkedin.common.Time", "com.linkedin.common.Urn", { "type" : "typeref", - "name" : "Pair", - "namespace" : "com.linkedin.common", + "name" : "VersionedUrn", + "namespace" : "com.linkedin.common.versioned", "ref" : "string", "java" : { - "class" : "com.linkedin.util.Pair", - "coercerClass" : "com.linkedin.util.PairCoercer" + "class" : "com.linkedin.common.urn.VersionedUrn", + "coercerClass" : "com.linkedin.util.VersionedUrnCoercer" } - }, "com.linkedin.common.Time", "com.linkedin.common.Urn", { + }, { "type" : "record", "name" : "Aspect", "namespace" : "com.linkedin.entity", @@ -117,7 +117,7 @@ "collection" : { "identifier" : { "name" : "entitiesVersionedV2Id", - "type" : "com.linkedin.common.Pair" + "type" : "com.linkedin.common.versioned.VersionedUrn" }, "supports" : [ "batch_get" ], "methods" : [ { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index 177ddc3dc0e11..2e243d51e7d9c 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -4570,7 +4570,10 @@ "fields" : [ { "name" : "displayName", "type" : "string", - "doc" : "Display name of the Policy" + "doc" : "Display name of the Policy", + "Searchable" : { + "fieldType" : "KEYWORD" + } }, { "name" : "description", "type" : "string", diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index b619010bd0c9f..cec9fba29b2c7 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -5,6 +5,7 @@ import com.linkedin.common.VersionedUrn; import com.linkedin.common.client.BaseClient; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; @@ -57,7 +58,7 @@ import com.linkedin.restli.client.RestLiResponseException; import com.linkedin.restli.common.HttpStatus; import com.linkedin.util.Pair; -import com.linkedin.util.PairCoercer; +import com.linkedin.util.VersionedUrnCoercer; import java.net.URISyntaxException; import java.util.Collection; import java.util.HashMap; @@ -198,23 +199,15 @@ public Map batchGetVersionedV2( .aspectsParam(aspectNames) .entityTypeParam(entityName) .ids(versionedUrns.stream() - .map(versionedUrn -> new Pair<>(versionedUrn.getUrn(), versionedUrn.getVersionStamp())) - .map(Pair::toString) - .map(string -> new PairCoercer().coerceOutput(string)).collect( - Collectors.toSet())); + .map(versionedUrn -> com.linkedin.common.urn.VersionedUrn.of(versionedUrn.getUrn().toString(), versionedUrn.getVersionStamp())) + .collect(Collectors.toSet())); return sendClientRequest(requestBuilder, authentication).getEntity() .getResults() .entrySet() .stream() - .collect(Collectors.toMap(entry -> { - try { - return Urn.createFromString((String) entry.getKey().getFirst()); - } catch (URISyntaxException e) { - throw new RuntimeException( - String.format("Failed to bind urn string with value %s into urn", entry.getKey())); - } - }, entry -> entry.getValue().getEntity())); + .collect(Collectors.toMap(entry -> + UrnUtils.getUrn(entry.getKey().getUrn()), entry -> entry.getValue().getEntity())); } /** diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java index 7c6250477c4af..3e226ed1574e1 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java @@ -13,7 +13,6 @@ import com.linkedin.restli.server.annotations.RestLiCollection; import com.linkedin.restli.server.annotations.RestMethod; import com.linkedin.restli.server.resources.CollectionResourceTaskTemplate; -import com.linkedin.util.Pair; import io.opentelemetry.extension.annotations.WithSpan; import java.util.Arrays; import java.util.Collections; @@ -36,8 +35,8 @@ */ @Slf4j @RestLiCollection(name = "entitiesVersionedV2", namespace = "com.linkedin.entity", - keyTyperefClass = com.linkedin.common.Pair.class) -public class EntityVersionedV2Resource extends CollectionResourceTaskTemplate, EntityResponse> { + keyTyperefClass = com.linkedin.common.versioned.VersionedUrn.class) +public class EntityVersionedV2Resource extends CollectionResourceTaskTemplate { @Inject @Named("entityService") @@ -47,7 +46,7 @@ public class EntityVersionedV2Resource extends CollectionResourceTaskTemplate> batchGetVersioned( - @Nonnull Set> versionedUrnStrs, + @Nonnull Set versionedUrnStrs, @QueryParam(PARAM_ENTITY_TYPE) @Nonnull String entityType, @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) { log.debug("BATCH GET VERSIONED V2 {}", versionedUrnStrs); @@ -59,10 +58,10 @@ public Task> batchGetVersioned( aspectNames == null ? getAllAspectNames(_entityService, entityType) : new HashSet<>(Arrays.asList(aspectNames)); try { return _entityService.getEntitiesVersionedV2(versionedUrnStrs.stream() - .map(pair -> { - VersionedUrn versionedUrn = new VersionedUrn().setUrn(UrnUtils.getUrn(pair.getFirst())); - if (pair.getSecond() != null) { - versionedUrn.setVersionStamp(pair.getSecond()); + .map(versionedUrnTyperef -> { + VersionedUrn versionedUrn = new VersionedUrn().setUrn(UrnUtils.getUrn(versionedUrnTyperef.getUrn())); + if (versionedUrnTyperef.getVersionStamp() != null) { + versionedUrn.setVersionStamp(versionedUrnTyperef.getVersionStamp()); } return versionedUrn; }).collect(Collectors.toSet()), projectedAspects); From 7f16a7617c315d9b5d81adc97ca95cd11740f2b5 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 28 Apr 2022 16:17:35 -0500 Subject: [PATCH 19/21] checkstyle --- .../java/com/linkedin/entity/client/RestliEntityClient.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index cec9fba29b2c7..27defaf417e0e 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -57,8 +57,6 @@ import com.linkedin.restli.client.Client; import com.linkedin.restli.client.RestLiResponseException; import com.linkedin.restli.common.HttpStatus; -import com.linkedin.util.Pair; -import com.linkedin.util.VersionedUrnCoercer; import java.net.URISyntaxException; import java.util.Collection; import java.util.HashMap; From 03ad8c22f6573247d5e1808218f3e9364c184b77 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 28 Apr 2022 18:28:38 -0500 Subject: [PATCH 20/21] feat(graphql): versioned dataset query - review comments: separate query --- .../datahub/graphql/GmsGraphQLEngine.java | 17 +- .../graphql/types/dataset/DatasetType.java | 23 +-- .../types/dataset/VersionedDatasetType.java | 98 +++++++++++ .../mappers/VersionedDatasetMapper.java | 159 ++++++++++++++++++ .../src/main/resources/entity.graphql | 122 +++++++++++++- .../common/urn/VersionedUrnUtils.java | 43 +++++ .../common/util/VersionedUrnUtilsTest.java | 32 ++++ .../linkedin/metadata/entity/EntityUtils.java | 34 ---- .../cassandra/CassandraEntityService.java | 3 +- .../entity/ebean/EbeanEntityService.java | 3 +- .../timeline/ebean/EbeanTimelineService.java | 2 +- 11 files changed, 475 insertions(+), 61 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java create mode 100644 li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrnUtils.java create mode 100644 li-utils/src/test/java/com/linkedin/common/util/VersionedUrnUtilsTest.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 000e8470a5ecf..2bc9f7ffb1834 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -143,6 +143,7 @@ import com.linkedin.datahub.graphql.types.corpgroup.CorpGroupType; import com.linkedin.datahub.graphql.types.corpuser.CorpUserType; import com.linkedin.datahub.graphql.types.dashboard.DashboardType; +import com.linkedin.datahub.graphql.types.dataset.VersionedDatasetType; import com.linkedin.datahub.graphql.types.notebook.NotebookType; import com.linkedin.datahub.graphql.types.dataflow.DataFlowType; import com.linkedin.datahub.graphql.types.datajob.DataJobType; @@ -241,6 +242,7 @@ public class GmsGraphQLEngine { private final DomainType domainType; private final NotebookType notebookType; private final AssertionType assertionType; + private final VersionedDatasetType versionedDatasetType; /** @@ -327,6 +329,7 @@ public GmsGraphQLEngine( this.domainType = new DomainType(entityClient); this.notebookType = new NotebookType(entityClient); this.assertionType = new AssertionType(entityClient); + this.versionedDatasetType = new VersionedDatasetType(entityClient); // Init Lists this.entityTypes = ImmutableList.of( @@ -348,7 +351,8 @@ public GmsGraphQLEngine( containerType, notebookType, domainType, - assertionType + assertionType, + versionedDatasetType ); this.loadableTypes = new ArrayList<>(entityTypes); this.ownerTypes = ImmutableList.of(corpUserType, corpGroupType); @@ -479,9 +483,10 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("autoCompleteForMultiple", new AutoCompleteForMultipleResolver(searchableTypes)) .dataFetcher("browse", new BrowseResolver(browsableTypes)) .dataFetcher("browsePaths", new BrowsePathsResolver(browsableTypes)) - .dataFetcher("dataset", getResolver(datasetType, - (env) -> new VersionedUrn().setUrn(UrnUtils.getUrn(env.getArgument(URN_FIELD_NAME))) - .setVersionStamp(env.getArgument(VERSION_STAMP_FIELD_NAME)))) + .dataFetcher("dataset", getResolver(datasetType)) + .dataFetcher("versionedDataset", getResolver(versionedDatasetType, + (env) -> new VersionedUrn().setUrn(UrnUtils.getUrn(env.getArgument(URN_FIELD_NAME))) + .setVersionStamp(env.getArgument(VERSION_STAMP_FIELD_NAME)))) .dataFetcher("notebook", getResolver(notebookType)) .dataFetcher("corpUser", getResolver(corpUserType)) .dataFetcher("corpGroup", getResolver(corpGroupType)) @@ -688,9 +693,7 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { ) .type("ForeignKeyConstraint", typeWiring -> typeWiring .dataFetcher("foreignDataset", new LoadableTypeResolver<>(datasetType, - (env) -> new VersionedUrn().setUrn( - UrnUtils.getUrn( - ((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn())))) + (env) -> ((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn())) ) .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring .dataFetcher("author", new LoadableTypeResolver<>(corpUserType, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index f248df4273deb..3be9551bcd322 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -2,7 +2,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -56,7 +55,7 @@ import static com.linkedin.metadata.Constants.*; -public class DatasetType implements SearchableEntityType, BrowsableEntityType, +public class DatasetType implements SearchableEntityType, BrowsableEntityType, MutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( @@ -104,24 +103,28 @@ public EntityType type() { } @Override - public Function getKeyProvider() { - return entity -> new VersionedUrn().setUrn(UrnUtils.getUrn(entity.getUrn())); + public Function getKeyProvider() { + return Entity::getUrn; } @Override - public List> batchLoad(@Nonnull final List versionedUrns, + public List> batchLoad(@Nonnull final List urnStrs, @Nonnull final QueryContext context) { try { + final List urns = urnStrs.stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + final Map datasetMap = - _entityClient.batchGetVersionedV2( + _entityClient.batchGetV2( Constants.DATASET_ENTITY_NAME, - new HashSet<>(versionedUrns), + new HashSet<>(urns), ASPECTS_TO_RESOLVE, context.getAuthentication()); final List gmsResults = new ArrayList<>(); - for (VersionedUrn versionedUrn : versionedUrns) { - gmsResults.add(datasetMap.getOrDefault(versionedUrn.getUrn(), null)); + for (Urn urn : urns) { + gmsResults.add(datasetMap.getOrDefault(urn, null)); } return gmsResults.stream() .map(gmsDataset -> gmsDataset == null ? null : DataFetcherResult.newResult() @@ -192,7 +195,7 @@ public Dataset update(@Nonnull String urn, @Nonnull DatasetUpdateInput input, @N throw new RuntimeException(String.format("Failed to write entity with urn %s", urn), e); } - return load(new VersionedUrn().setUrn(UrnUtils.getUrn(urn)), context).getData(); + return load(urn, context).getData(); } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java new file mode 100644 index 0000000000000..a8f61232a9aec --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java @@ -0,0 +1,98 @@ +package com.linkedin.datahub.graphql.types.dataset; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.VersionedUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.VersionedDataset; +import com.linkedin.datahub.graphql.types.dataset.mappers.VersionedDatasetMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +import static com.linkedin.metadata.Constants.*; + + +public class VersionedDatasetType implements com.linkedin.datahub.graphql.types.EntityType { + + private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( + DATASET_KEY_ASPECT_NAME, + DATASET_PROPERTIES_ASPECT_NAME, + EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, + DATASET_DEPRECATION_ASPECT_NAME, // This aspect is deprecated. + DEPRECATION_ASPECT_NAME, + DATASET_UPSTREAM_LINEAGE_ASPECT_NAME, + UPSTREAM_LINEAGE_ASPECT_NAME, + EDITABLE_SCHEMA_METADATA_ASPECT_NAME, + VIEW_PROPERTIES_ASPECT_NAME, + OWNERSHIP_ASPECT_NAME, + INSTITUTIONAL_MEMORY_ASPECT_NAME, + GLOBAL_TAGS_ASPECT_NAME, + GLOSSARY_TERMS_ASPECT_NAME, + STATUS_ASPECT_NAME, + CONTAINER_ASPECT_NAME, + DOMAINS_ASPECT_NAME, + SCHEMA_METADATA_ASPECT_NAME + ); + + private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); + private static final String ENTITY_NAME = "dataset"; + + private final EntityClient _entityClient; + + public VersionedDatasetType(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public Class objectClass() { + return VersionedDataset.class; + } + + @Override + public EntityType type() { + return EntityType.DATASET; + } + + @Override + public Function getKeyProvider() { + return entity -> new VersionedUrn().setUrn(UrnUtils.getUrn(entity.getUrn())); + } + + @Override + public List> batchLoad(@Nonnull final List versionedUrns, + @Nonnull final QueryContext context) { + try { + final Map datasetMap = + _entityClient.batchGetVersionedV2( + Constants.DATASET_ENTITY_NAME, + new HashSet<>(versionedUrns), + ASPECTS_TO_RESOLVE, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (VersionedUrn versionedUrn : versionedUrns) { + gmsResults.add(datasetMap.getOrDefault(versionedUrn.getUrn(), null)); + } + return gmsResults.stream() + .map(gmsDataset -> gmsDataset == null ? null : DataFetcherResult.newResult() + .data(VersionedDatasetMapper.map(gmsDataset)) + .build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Datasets", e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java new file mode 100644 index 0000000000000..5f9d4944e5ae2 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java @@ -0,0 +1,159 @@ +package com.linkedin.datahub.graphql.types.dataset.mappers; + +import com.linkedin.common.Deprecation; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.InstitutionalMemory; +import com.linkedin.common.Ownership; +import com.linkedin.common.Status; +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.generated.Container; +import com.linkedin.datahub.graphql.generated.DataPlatform; +import com.linkedin.datahub.graphql.generated.DatasetEditableProperties; +import com.linkedin.datahub.graphql.generated.Domain; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FabricType; +import com.linkedin.datahub.graphql.generated.VersionedDataset; +import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; +import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; +import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; +import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; +import com.linkedin.dataset.DatasetDeprecation; +import com.linkedin.dataset.DatasetProperties; +import com.linkedin.dataset.EditableDatasetProperties; +import com.linkedin.dataset.ViewProperties; +import com.linkedin.domain.Domains; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.metadata.key.DatasetKey; +import com.linkedin.schema.EditableSchemaMetadata; +import com.linkedin.schema.SchemaMetadata; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.Constants.*; + + +/** + * Maps GMS response objects to objects conforming to the GQL schema. + * + * To be replaced by auto-generated mappers implementations + */ +@Slf4j +public class VersionedDatasetMapper implements ModelMapper { + + public static final VersionedDatasetMapper INSTANCE = new VersionedDatasetMapper(); + + public static VersionedDataset map(@Nonnull final EntityResponse dataset) { + return INSTANCE.apply(dataset); + } + + @Override + public VersionedDataset apply(@Nonnull final EntityResponse entityResponse) { + VersionedDataset result = new VersionedDataset(); + result.setUrn(entityResponse.getUrn().toString()); + result.setType(EntityType.DATASET); + + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult(DATASET_KEY_ASPECT_NAME, this::mapDatasetKey); + mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, this::mapDatasetProperties); + mappingHelper.mapToResult(DATASET_DEPRECATION_ASPECT_NAME, (dataset, dataMap) -> + dataset.setDeprecation(DatasetDeprecationMapper.map(new DatasetDeprecation(dataMap)))); + mappingHelper.mapToResult(SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) -> + dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap)))); + mappingHelper.mapToResult(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, this::mapEditableDatasetProperties); + mappingHelper.mapToResult(VIEW_PROPERTIES_ASPECT_NAME, this::mapViewProperties); + mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dataset, dataMap) -> + dataset.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); + mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (dataset, dataMap) -> + dataset.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + mappingHelper.mapToResult(STATUS_ASPECT_NAME, (dataset, dataMap) -> + dataset.setStatus(StatusMapper.map(new Status(dataMap)))); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(EDITABLE_SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) -> + dataset.setEditableSchemaMetadata(EditableSchemaMetadataMapper.map(new EditableSchemaMetadata(dataMap)))); + mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (dataset, dataMap) -> + dataset.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + mappingHelper.mapToResult(CONTAINER_ASPECT_NAME, this::mapContainers); + mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); + mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (dataset, dataMap) -> + dataset.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); + + return mappingHelper.getResult(); + } + + private void mapDatasetKey(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + final DatasetKey gmsKey = new DatasetKey(dataMap); + dataset.setName(gmsKey.getName()); + dataset.setOrigin(FabricType.valueOf(gmsKey.getOrigin().toString())); + dataset.setPlatform(DataPlatform.builder() + .setType(EntityType.DATA_PLATFORM) + .setUrn(gmsKey.getPlatform().toString()).build()); + } + + private void mapDatasetProperties(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + final DatasetProperties gmsProperties = new DatasetProperties(dataMap); + final com.linkedin.datahub.graphql.generated.DatasetProperties properties = + new com.linkedin.datahub.graphql.generated.DatasetProperties(); + properties.setDescription(gmsProperties.getDescription()); + properties.setOrigin(dataset.getOrigin()); + if (gmsProperties.getExternalUrl() != null) { + properties.setExternalUrl(gmsProperties.getExternalUrl().toString()); + } + properties.setCustomProperties(StringMapMapper.map(gmsProperties.getCustomProperties())); + if (gmsProperties.getName() != null) { + properties.setName(gmsProperties.getName()); + } else { + properties.setName(dataset.getName()); + } + properties.setQualifiedName(gmsProperties.getQualifiedName()); + dataset.setProperties(properties); + } + + private void mapEditableDatasetProperties(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + final EditableDatasetProperties editableDatasetProperties = new EditableDatasetProperties(dataMap); + final DatasetEditableProperties editableProperties = new DatasetEditableProperties(); + editableProperties.setDescription(editableDatasetProperties.getDescription()); + dataset.setEditableProperties(editableProperties); + } + + private void mapViewProperties(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + final ViewProperties properties = new ViewProperties(dataMap); + final com.linkedin.datahub.graphql.generated.ViewProperties graphqlProperties = + new com.linkedin.datahub.graphql.generated.ViewProperties(); + graphqlProperties.setMaterialized(properties.isMaterialized()); + graphqlProperties.setLanguage(properties.getViewLanguage()); + graphqlProperties.setLogic(properties.getViewLogic()); + dataset.setViewProperties(graphqlProperties); + } + + private void mapGlobalTags(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap)); + dataset.setTags(globalTags); + } + + private void mapContainers(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + final com.linkedin.container.Container gmsContainer = new com.linkedin.container.Container(dataMap); + dataset.setContainer(Container + .builder() + .setType(EntityType.CONTAINER) + .setUrn(gmsContainer.getContainer().toString()) + .build()); + } + + private void mapDomains(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + final Domains domains = new Domains(dataMap); + // Currently we only take the first domain if it exists. + if (domains.getDomains().size() > 0) { + dataset.setDomain(Domain.builder() + .setType(EntityType.DOMAIN) + .setUrn(domains.getDomains().get(0).toString()).build()); + } + } +} diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index e9b07eb600bd5..8bc6b086d1c09 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -32,7 +32,12 @@ type Query { """ Fetch a Dataset by primary key (urn) """ - dataset(urn: String!, versionStamp: String): Dataset + dataset(urn: String!): Dataset + + """ + Fetch a Dataset by primary key (urn) at a point in time based on aspect versions (versionStamp) + """ + versionedDataset(urn: String!, versionStamp: String): VersionedDataset """ Fetch a Dashboard by primary key (urn) @@ -331,11 +336,6 @@ interface Entity { A standard Entity Type """ type: EntityType! - - """ - List of relationships between the source Entity and some destination entities with a given types - """ - relationships(input: RelationshipsInput!): EntityRelationshipsResult } """ @@ -800,7 +800,7 @@ type Dataset implements EntityWithRelationships & Entity { """ Schema metadata of the dataset """ - schema: Schema + schema: Schema @deprecated(reason: "Use `schemaMetadata`") """ Deprecated, use properties field instead @@ -862,6 +862,114 @@ type Dataset implements EntityWithRelationships & Entity { runs(start: Int, count: Int, direction: RelationshipDirection!): DataProcessInstanceResult } +""" +A Dataset entity, which encompasses Relational Tables, Document store collections, streaming topics, and other sets of data having an independent lifecycle +""" +type VersionedDataset implements Entity { + """ + The primary key of the Dataset + """ + urn: String! + + """ + The standard Entity Type + """ + type: EntityType! + + """ + Standardized platform urn where the dataset is defined + """ + platform: DataPlatform! + + """ + The parent container in which the entity resides + """ + container: Container + + """ + Unique guid for dataset + No longer to be used as the Dataset display name. Use properties.name instead + """ + name: String! + + """ + An additional set of read only properties + """ + properties: DatasetProperties + + """ + An additional set of of read write properties + """ + editableProperties: DatasetEditableProperties + + """ + Ownership metadata of the dataset + """ + ownership: Ownership + + """ + The deprecation status of the dataset + """ + deprecation: Deprecation + + """ + References to internal resources related to the dataset + """ + institutionalMemory: InstitutionalMemory + + """ + Editable schema metadata of the dataset + """ + editableSchemaMetadata: EditableSchemaMetadata + + """ + Status of the Dataset + """ + status: Status + + """ + Tags used for searching dataset + """ + tags: GlobalTags + + """ + The structured glossary terms associated with the dataset + """ + glossaryTerms: GlossaryTerms + + """ + The Domain associated with the Dataset + """ + domain: Domain + + """ + Experimental! The resolved health status of the Dataset + """ + health: Health + + """ + Schema metadata of the dataset + """ + schema: Schema + + """ + Sub Types that this entity implements + """ + subTypes: SubTypes + + """ + View related properties. Only relevant if subtypes field contains view. + """ + viewProperties: ViewProperties + + """ + Deprecated, see the properties field instead + Environment in which the dataset belongs to or where it was generated + Note that this field will soon be deprecated in favor of a more standardized concept of Environment + """ + origin: FabricType! @deprecated +} + """ Data Process instances that match the provided query """ diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrnUtils.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrnUtils.java new file mode 100644 index 0000000000000..f7e0b6c99e334 --- /dev/null +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/VersionedUrnUtils.java @@ -0,0 +1,43 @@ +package com.linkedin.common.urn; + +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import org.apache.commons.lang3.StringUtils; + + +public class VersionedUrnUtils { + + private VersionedUrnUtils() { + } + + public static Map convertVersionStamp(String versionStamp) { + Map aspectVersionMap = new HashMap<>(); + if (StringUtils.isBlank(versionStamp)) { + return aspectVersionMap; + } + String[] aspectNameVersionPairs = versionStamp.split(";"); + for (String pair : aspectNameVersionPairs) { + String[] tokens = pair.split(":"); + if (tokens.length != 2) { + throw new IllegalArgumentException("Invalid version stamp cannot be parsed: " + versionStamp); + } + try { + aspectVersionMap.put(tokens[0], Long.valueOf(tokens[1])); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid value for aspect version: " + tokens[1]); + } + } + + return aspectVersionMap; + } + + public static String constructVersionStamp(SortedMap versionStampMap) { + StringBuilder versionStamp = versionStampMap.entrySet().stream() + .collect(StringBuilder::new, (builder, entry) -> builder.append(entry.getKey()) + .append(":") + .append(entry.getValue()).append(";"), StringBuilder::append); + // trim off last ; + return versionStamp.substring(0, versionStamp.length() - 1); + } +} diff --git a/li-utils/src/test/java/com/linkedin/common/util/VersionedUrnUtilsTest.java b/li-utils/src/test/java/com/linkedin/common/util/VersionedUrnUtilsTest.java new file mode 100644 index 0000000000000..cb5ac62d71a1d --- /dev/null +++ b/li-utils/src/test/java/com/linkedin/common/util/VersionedUrnUtilsTest.java @@ -0,0 +1,32 @@ +package com.linkedin.common.util; + +import com.linkedin.common.urn.VersionedUrnUtils; +import java.util.Comparator; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.*; + + +public class VersionedUrnUtilsTest { + + private static final String SCHEMA_METADATA = "schemaMetadata"; + private static final String DATASET_KEY = "datasetKey"; + private static final String SOME_ASPECT = "someAspect"; + + @Test + public void testVersionStampConstructConvert() { + SortedMap sortedMap = new TreeMap<>(Comparator.naturalOrder()); + sortedMap.put(SCHEMA_METADATA, 15L); + sortedMap.put(DATASET_KEY, 0L); + sortedMap.put(SOME_ASPECT, 111231242456L); + String versionStamp = VersionedUrnUtils.constructVersionStamp(sortedMap); + + Map versionStampMap = VersionedUrnUtils.convertVersionStamp(versionStamp); + assertEquals(versionStampMap.get(SCHEMA_METADATA), sortedMap.get(SCHEMA_METADATA)); + assertEquals(versionStampMap.get(DATASET_KEY), sortedMap.get(DATASET_KEY)); + assertEquals(versionStampMap.get(SOME_ASPECT), sortedMap.get(SOME_ASPECT)); + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index a3730118c4fa5..7789bc4c3c1eb 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -7,16 +7,12 @@ import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.EnvelopedAspect; -import java.util.HashMap; -import java.util.Map; -import java.util.SortedMap; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.SystemMetadata; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; import javax.annotation.Nonnull; @@ -94,34 +90,4 @@ public static boolean checkIfRemoved(EntityService entityService, Urn entityUrn) return false; } } - - public static Map convertVersionStamp(String versionStamp) { - Map aspectVersionMap = new HashMap<>(); - if (StringUtils.isBlank(versionStamp)) { - return aspectVersionMap; - } - String[] aspectNameVersionPairs = versionStamp.split(";"); - for (String pair : aspectNameVersionPairs) { - String[] tokens = pair.split(":"); - if (tokens.length != 2) { - throw new IllegalArgumentException("Invalid version stamp cannot be parsed: " + versionStamp); - } - try { - aspectVersionMap.put(tokens[0], Long.valueOf(tokens[1])); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid value for aspect version: " + tokens[1]); - } - } - - return aspectVersionMap; - } - - public static String constructVersionStamp(SortedMap versionStampMap) { - StringBuilder versionStamp = versionStampMap.entrySet().stream() - .collect(StringBuilder::new, (builder, entry) -> builder.append(entry.getKey()) - .append(":") - .append(entry.getValue()).append(";"), StringBuilder::append); - // trim off last ; - return versionStamp.substring(0, versionStamp.length() - 1); - } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java index 04356925f1fdb..7a529e7598522 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraEntityService.java @@ -11,6 +11,7 @@ import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.common.urn.VersionedUrnUtils; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; @@ -185,7 +186,7 @@ public Map> getVersionedEnvelopedAspects( Map> urnAspectVersionMap = versionedUrns.stream() .collect(Collectors.toMap(versionedUrn -> versionedUrn.getUrn().toString(), - versionedUrn -> EntityUtils.convertVersionStamp(versionedUrn.getVersionStamp()))); + versionedUrn -> VersionedUrnUtils.convertVersionStamp(versionedUrn.getVersionStamp()))); // Cover full/partial versionStamp final Set dbKeys = urnAspectVersionMap.entrySet().stream() diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 28d2e56c2a32b..ded5c6dd14318 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -11,6 +11,7 @@ import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.common.urn.VersionedUrnUtils; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.JacksonDataTemplateCodec; @@ -212,7 +213,7 @@ public Map> getVersionedEnvelopedAspects( Map> urnAspectVersionMap = versionedUrns.stream() .collect(Collectors.toMap(versionedUrn -> versionedUrn.getUrn().toString(), - versionedUrn -> EntityUtils.convertVersionStamp(versionedUrn.getVersionStamp()))); + versionedUrn -> VersionedUrnUtils.convertVersionStamp(versionedUrn.getVersionStamp()))); // Cover full/partial versionStamp final Set dbKeys = urnAspectVersionMap.entrySet().stream() diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java index d0c8d10eb25a4..6ac150bf9917d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/ebean/EbeanTimelineService.java @@ -45,7 +45,7 @@ import org.apache.parquet.SemanticVersion; import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.entity.EntityUtils.*; +import static com.linkedin.common.urn.VersionedUrnUtils.*; public class EbeanTimelineService implements TimelineService { From 988ddc81c67d87caff862d6e54705c6f64c2ab15 Mon Sep 17 00:00:00 2001 From: Ryan Holstien Date: Thu, 28 Apr 2022 19:54:04 -0500 Subject: [PATCH 21/21] feat(graphql): versioned dataset query - add relationship no-op --- .../linkedin/datahub/graphql/GmsGraphQLEngine.java | 12 ++++++++++++ .../src/main/resources/entity.graphql | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 2bc9f7ffb1834..600dabda56b52 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -172,6 +172,7 @@ import graphql.execution.DataFetcherResult; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import graphql.schema.StaticDataFetcher; import graphql.schema.idl.RuntimeWiring; import java.io.IOException; import java.io.InputStream; @@ -404,6 +405,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureAssertionResolvers(builder); configurePolicyResolvers(builder); configureDataProcessInstanceResolvers(builder); + configureVersionedDatasetResolvers(builder); } public GraphQLEngine.Builder builder() { @@ -702,6 +704,16 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { } + /** + * Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.VersionedDataset} type. + */ + private void configureVersionedDatasetResolvers(final RuntimeWiring.Builder builder) { + builder + .type("VersionedDataset", typeWiring -> typeWiring + .dataFetcher("relationships", new StaticDataFetcher(null))); + + } + private void configureGlossaryTermResolvers(final RuntimeWiring.Builder builder) { builder.type("GlossaryTerm", typeWiring -> typeWiring .dataFetcher("schemaMetadata", new AspectResolver()) diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 8bc6b086d1c09..2dad0a552d26e 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -336,6 +336,11 @@ interface Entity { A standard Entity Type """ type: EntityType! + + """ + List of relationships between the source Entity and some destination entities with a given types + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult } """ @@ -968,6 +973,11 @@ type VersionedDataset implements Entity { Note that this field will soon be deprecated in favor of a more standardized concept of Environment """ origin: FabricType! @deprecated + + """ + No-op, has to be included due to model + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult @deprecated } """