Skip to content

Commit

Permalink
Merge branch 'master' into tableau-retries
Browse files Browse the repository at this point in the history
  • Loading branch information
@hsheth2
hsheth2 committed Jul 30, 2024
2 parents c94e8e3 + 9321e94 commit b714311
Show file tree
Hide file tree
Showing 179 changed files with 5,505 additions and 2,433 deletions.
1 change: 1 addition & 0 deletions datahub-graphql-core/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ dependencies {
implementation externalDependency.opentelemetryAnnotations

implementation externalDependency.slf4jApi
implementation externalDependency.springContext
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok

Expand Down
1 change: 1 addition & 0 deletions datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ private Constants() {}
public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
public static final String FORMS_SCHEMA_FILE = "forms.graphql";
public static final String ASSERTIONS_SCHEMA_FILE = "assertions.graphql";
public static final String COMMON_SCHEMA_FILE = "common.graphql";
public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql";
public static final String CONTRACTS_SCHEMA_FILE = "contract.graphql";
public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql";
Expand Down
45 changes: 38 additions & 7 deletions datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
import com.linkedin.datahub.graphql.generated.MLPrimaryKey;
import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties;
import com.linkedin.datahub.graphql.generated.MatchedField;
import com.linkedin.datahub.graphql.generated.MetadataAttribution;
import com.linkedin.datahub.graphql.generated.Notebook;
import com.linkedin.datahub.graphql.generated.Owner;
import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity;
Expand Down Expand Up @@ -284,6 +285,8 @@
import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossEntitiesResolver;
import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver;
import com.linkedin.datahub.graphql.resolvers.search.SearchResolver;
import com.linkedin.datahub.graphql.resolvers.settings.docPropagation.DocPropagationSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.docPropagation.UpdateDocPropagationSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.view.GlobalViewsSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver;
Expand Down Expand Up @@ -695,7 +698,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
businessAttributeType));
this.loadableTypes = new ArrayList<>(entityTypes);
// Extend loadable types with types from the plugins
// This allows us to offer search and browse capabilities out of the box for those types
// This allows us to offer search and browse capabilities out of the box for
// those types
for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
this.entityTypes.addAll(plugin.getEntityTypes());
Collection<? extends LoadableType<?, ?>> pluginLoadableTypes = plugin.getLoadableTypes();
Expand Down Expand Up @@ -790,6 +794,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
configureBusinessAttributeAssociationResolver(builder);
configureConnectionResolvers(builder);
configureDeprecationResolvers(builder);
configureMetadataAttributionResolver(builder);
}

private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
Expand Down Expand Up @@ -843,7 +848,8 @@ public GraphQLEngine.Builder builder() {
.addSchema(fileBasedSchema(CONNECTIONS_SCHEMA_FILE))
.addSchema(fileBasedSchema(ASSERTIONS_SCHEMA_FILE))
.addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE))
.addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE));
.addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE))
.addSchema(fileBasedSchema(COMMON_SCHEMA_FILE));

for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
List<String> pluginSchemaFiles = plugin.getSchemaFiles();
Expand Down Expand Up @@ -1087,8 +1093,10 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
new BrowseV2Resolver(this.entityClient, this.viewService, this.formService))
.dataFetcher("businessAttribute", getResolver(businessAttributeType))
.dataFetcher(
"listBusinessAttributes",
new ListBusinessAttributesResolver(this.entityClient)));
"listBusinessAttributes", new ListBusinessAttributesResolver(this.entityClient))
.dataFetcher(
"docPropagationSettings",
new DocPropagationSettingsResolver(this.settingsService)));
}

private DataFetcher getEntitiesResolver() {
Expand Down Expand Up @@ -1340,7 +1348,11 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher(
"createForm", new CreateFormResolver(this.entityClient, this.formService))
.dataFetcher("deleteForm", new DeleteFormResolver(this.entityClient))
.dataFetcher("updateForm", new UpdateFormResolver(this.entityClient));
.dataFetcher("updateForm", new UpdateFormResolver(this.entityClient))
.dataFetcher(
"updateDocPropagationSettings",
new UpdateDocPropagationSettingsResolver(this.settingsService));

if (featureFlags.isBusinessAttributeEntityEnabled()) {
typeWiring
.dataFetcher(
Expand Down Expand Up @@ -1869,7 +1881,9 @@ private void configureCorpGroupResolvers(final RuntimeWiring.Builder builder) {
"CorpGroup",
typeWiring ->
typeWiring
.dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
.dataFetcher(
"relationships",
new EntityRelationshipsResultResolver(graphClient, entityService))
.dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
.dataFetcher(
"aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
Expand Down Expand Up @@ -2821,7 +2835,8 @@ private void configureContractResolvers(final RuntimeWiring.Builder builder) {
}

private void configurePolicyResolvers(final RuntimeWiring.Builder builder) {
// Register resolvers for "resolvedUsers" and "resolvedGroups" field of the Policy type.
// Register resolvers for "resolvedUsers" and "resolvedGroups" field of the
// Policy type.
builder.type(
"ActorFilter",
typeWiring ->
Expand Down Expand Up @@ -3174,4 +3189,20 @@ private void configureDeprecationResolvers(final RuntimeWiring.Builder builder)
new EntityTypeResolver(
entityTypes, (env) -> ((Deprecation) env.getSource()).getActorEntity())));
}

private void configureMetadataAttributionResolver(final RuntimeWiring.Builder builder) {
builder.type(
"MetadataAttribution",
typeWiring ->
typeWiring
.dataFetcher(
"actor",
new EntityTypeResolver(
entityTypes, (env) -> ((MetadataAttribution) env.getSource()).getActor()))
.dataFetcher(
"source",
new EntityTypeResolver(
entityTypes,
(env) -> ((MetadataAttribution) env.getSource()).getSource())));
}
}
5 changes: 5 additions & 0 deletions ...hql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,11 @@ public static boolean canManageForms(@Nonnull QueryContext context) {
PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE);
}

public static boolean canManageFeatures(@Nonnull QueryContext context) {
return AuthUtil.isAuthorized(
context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_FEATURES_PRIVILEGE);
}

public static boolean isAuthorized(
@Nonnull Authorizer authorizer,
@Nonnull String actor,
Expand Down
121 changes: 75 additions & 46 deletions ...a/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,16 @@
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
import java.time.DateTimeException;
import java.time.ZoneId;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.support.CronExpression;

/** Creates or updates an ingestion source. Requires the MANAGE_INGESTION privilege. */
@Slf4j
Expand All @@ -46,55 +50,51 @@ public UpsertIngestionSourceResolver(final EntityClient entityClient) {
public CompletableFuture<String> get(final DataFetchingEnvironment environment) throws Exception {
final QueryContext context = environment.getContext();

return GraphQLConcurrencyUtils.supplyAsync(
() -> {
if (IngestionAuthUtils.canManageIngestion(context)) {

final Optional<String> ingestionSourceUrn =
Optional.ofNullable(environment.getArgument("urn"));
final UpdateIngestionSourceInput input =
bindArgument(environment.getArgument("input"), UpdateIngestionSourceInput.class);
if (!IngestionAuthUtils.canManageIngestion(context)) {
throw new AuthorizationException(
"Unauthorized to perform this action. Please contact your DataHub administrator.");
}
final Optional<String> ingestionSourceUrn = Optional.ofNullable(environment.getArgument("urn"));
final UpdateIngestionSourceInput input =
bindArgument(environment.getArgument("input"), UpdateIngestionSourceInput.class);

// Create the policy info.
final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input);
final MetadataChangeProposal proposal;
if (ingestionSourceUrn.isPresent()) {
// Update existing ingestion source
try {
proposal =
buildMetadataChangeProposalWithUrn(
Urn.createFromString(ingestionSourceUrn.get()),
INGESTION_INFO_ASPECT_NAME,
info);
} catch (URISyntaxException e) {
throw new DataHubGraphQLException(
String.format("Malformed urn %s provided.", ingestionSourceUrn.get()),
DataHubGraphQLErrorCode.BAD_REQUEST);
}
} else {
// Create new ingestion source
// Since we are creating a new Ingestion Source, we need to generate a unique UUID.
final UUID uuid = UUID.randomUUID();
final String uuidStr = uuid.toString();
final DataHubIngestionSourceKey key = new DataHubIngestionSourceKey();
key.setId(uuidStr);
proposal =
buildMetadataChangeProposalWithKey(
key, INGESTION_SOURCE_ENTITY_NAME, INGESTION_INFO_ASPECT_NAME, info);
}
// Create the policy info.
final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input);
final MetadataChangeProposal proposal;
if (ingestionSourceUrn.isPresent()) {
// Update existing ingestion source
try {
proposal =
buildMetadataChangeProposalWithUrn(
Urn.createFromString(ingestionSourceUrn.get()), INGESTION_INFO_ASPECT_NAME, info);
} catch (URISyntaxException e) {
throw new DataHubGraphQLException(
String.format("Malformed urn %s provided.", ingestionSourceUrn.get()),
DataHubGraphQLErrorCode.BAD_REQUEST);
}
} else {
// Create new ingestion source
// Since we are creating a new Ingestion Source, we need to generate a unique UUID.
final UUID uuid = UUID.randomUUID();
final String uuidStr = uuid.toString();
final DataHubIngestionSourceKey key = new DataHubIngestionSourceKey();
key.setId(uuidStr);
proposal =
buildMetadataChangeProposalWithKey(
key, INGESTION_SOURCE_ENTITY_NAME, INGESTION_INFO_ASPECT_NAME, info);
}

try {
return _entityClient.ingestProposal(context.getOperationContext(), proposal, false);
} catch (Exception e) {
throw new RuntimeException(
String.format(
"Failed to perform update against ingestion source with urn %s",
input.toString()),
e);
}
return GraphQLConcurrencyUtils.supplyAsync(
() -> {
try {
return _entityClient.ingestProposal(context.getOperationContext(), proposal, false);
} catch (Exception e) {
throw new RuntimeException(
String.format(
"Failed to perform update against ingestion source with urn %s",
input.toString()),
e);
}
throw new AuthorizationException(
"Unauthorized to perform this action. Please contact your DataHub administrator.");
},
this.getClass().getSimpleName(),
"get");
Expand Down Expand Up @@ -137,9 +137,38 @@ private DataHubIngestionSourceConfig mapConfig(final UpdateIngestionSourceConfig

private DataHubIngestionSourceSchedule mapSchedule(
final UpdateIngestionSourceScheduleInput input) {

final String modifiedCronInterval = adjustCronInterval(input.getInterval());
try {
CronExpression.parse(modifiedCronInterval);
} catch (IllegalArgumentException e) {
throw new DataHubGraphQLException(
String.format("Invalid cron schedule `%s`: %s", input.getInterval(), e.getMessage()),
DataHubGraphQLErrorCode.BAD_REQUEST);
}
try {
ZoneId.of(input.getTimezone());
} catch (DateTimeException e) {
throw new DataHubGraphQLException(
String.format("Invalid timezone `%s`: %s", input.getTimezone(), e.getMessage()),
DataHubGraphQLErrorCode.BAD_REQUEST);
}

final DataHubIngestionSourceSchedule result = new DataHubIngestionSourceSchedule();
result.setInterval(input.getInterval());
result.setTimezone(input.getTimezone());
return result;
}

// Copied from IngestionScheduler.java
private String adjustCronInterval(final String origCronInterval) {
Objects.requireNonNull(origCronInterval, "origCronInterval must not be null");
// Typically we support 5-character cron. Spring's lib only supports 6 character cron so we make
// an adjustment here.
final String[] originalCronParts = origCronInterval.split(" ");
if (originalCronParts.length == 5) {
return String.format("0 %s", origCronInterval);
}
return origCronInterval;
}
}
35 changes: 32 additions & 3 deletions ...n/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,21 @@

import com.linkedin.common.EntityRelationship;
import com.linkedin.common.EntityRelationships;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.datahub.graphql.generated.EntityRelationshipsResult;
import com.linkedin.datahub.graphql.generated.RelationshipsInput;
import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.graph.GraphClient;
import com.linkedin.metadata.query.filter.RelationshipDirection;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
Expand All @@ -29,8 +32,16 @@ public class EntityRelationshipsResultResolver

private final GraphClient _graphClient;

private final EntityService _entityService;

public EntityRelationshipsResultResolver(final GraphClient graphClient) {
this(graphClient, null);
}

public EntityRelationshipsResultResolver(
final GraphClient graphClient, final EntityService entityService) {
_graphClient = graphClient;
_entityService = entityService;
}

@Override
Expand All @@ -47,13 +58,16 @@ public CompletableFuture<EntityRelationshipsResult> get(DataFetchingEnvironment
final Integer count = input.getCount(); // Optional!
final RelationshipDirection resolvedDirection =
RelationshipDirection.valueOf(relationshipDirection.toString());
final boolean includeSoftDelete = input.getIncludeSoftDelete();

return GraphQLConcurrencyUtils.supplyAsync(
() ->
mapEntityRelationships(
context,
fetchEntityRelationships(
urn, relationshipTypes, resolvedDirection, start, count, context.getActorUrn()),
resolvedDirection),
resolvedDirection,
includeSoftDelete),
this.getClass().getSimpleName(),
"get");
}
Expand All @@ -72,13 +86,28 @@ private EntityRelationships fetchEntityRelationships(
private EntityRelationshipsResult mapEntityRelationships(
@Nullable final QueryContext context,
final EntityRelationships entityRelationships,
final RelationshipDirection relationshipDirection) {
final RelationshipDirection relationshipDirection,
final boolean includeSoftDelete) {
final EntityRelationshipsResult result = new EntityRelationshipsResult();

final Set<Urn> existentUrns;
if (context != null && _entityService != null && !includeSoftDelete) {
Set<Urn> allRelatedUrns =
entityRelationships.getRelationships().stream()
.map(EntityRelationship::getEntity)
.collect(Collectors.toSet());
existentUrns = _entityService.exists(context.getOperationContext(), allRelatedUrns, false);
} else {
existentUrns = null;
}

List<EntityRelationship> viewable =
entityRelationships.getRelationships().stream()
.filter(
rel -> context == null || canView(context.getOperationContext(), rel.getEntity()))
rel ->
(existentUrns == null || existentUrns.contains(rel.getEntity()))
&& (context == null
|| canView(context.getOperationContext(), rel.getEntity())))
.collect(Collectors.toList());

result.setStart(entityRelationships.getStart());
Expand Down
Loading

0 comments on commit b714311

Please sign in to comment.