-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(graph): graph index soft-delete support
* record entity soft delete status on edges * upgrade job to backfill status on edges * filter graph service by soft-deleted entities
- Loading branch information
1 parent
315ff8f
commit a4d083f
Showing
66 changed files
with
2,149 additions
and
954 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
5 changes: 3 additions & 2 deletions
5
...nfig/ReindexDataJobViaNodesCLLConfig.java → ...raph/ReindexDataJobViaNodesCLLConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
...rade/src/main/java/com/linkedin/datahub/upgrade/config/graph/ReindexEdgeStatusConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package com.linkedin.datahub.upgrade.config.graph; | ||
|
|
||
| import com.linkedin.datahub.upgrade.config.SystemUpdateCondition; | ||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.datahub.upgrade.system.graph.edgestatus.ReindexEdgeStatus; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Conditional; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| @Configuration | ||
| @Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) | ||
| public class ReindexEdgeStatusConfig { | ||
|
|
||
| @Bean | ||
| public NonBlockingSystemUpgrade reindexEdgeStatus( | ||
| final OperationContext opContext, | ||
| final EntityService<?> entityService, | ||
| final AspectDao aspectDao, | ||
| @Value("${elasticsearch.search.graph.graphStatusEnabled}") final boolean featureEnabled, | ||
| @Value("${systemUpdate.edgeStatus.enabled}") final boolean enabled, | ||
| @Value("${systemUpdate.edgeStatus.batchSize}") final Integer batchSize, | ||
| @Value("${systemUpdate.edgeStatus.delayMs}") final Integer delayMs, | ||
| @Value("${systemUpdate.edgeStatus.limit}") final Integer limit) { | ||
| return new ReindexEdgeStatus( | ||
| opContext, entityService, aspectDao, featureEnabled && enabled, batchSize, delayMs, limit); | ||
| } | ||
| } |
50 changes: 50 additions & 0 deletions
50
...src/main/java/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexEdgeStatus.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| package com.linkedin.datahub.upgrade.system.graph.edgestatus; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import java.util.List; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** | ||
| * A job that reindexes all status aspects as part of the graph edges containing status information. | ||
| * This is required to make sure previously written status information is present in the graph | ||
| * index. | ||
| */ | ||
| @Slf4j | ||
| public class ReindexEdgeStatus implements NonBlockingSystemUpgrade { | ||
|
|
||
| private final List<UpgradeStep> _steps; | ||
|
|
||
| public ReindexEdgeStatus( | ||
| @Nonnull OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| AspectDao aspectDao, | ||
| boolean enabled, | ||
| Integer batchSize, | ||
| Integer batchDelayMs, | ||
| Integer limit) { | ||
| if (enabled) { | ||
| _steps = | ||
| ImmutableList.of( | ||
| new ReindexReindexEdgeStatusStep( | ||
| opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); | ||
| } else { | ||
| _steps = ImmutableList.of(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return this.getClass().getName(); | ||
| } | ||
|
|
||
| @Override | ||
| public List<UpgradeStep> steps() { | ||
| return _steps; | ||
| } | ||
| } |
56 changes: 56 additions & 0 deletions
56
...va/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexReindexEdgeStatusStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package com.linkedin.datahub.upgrade.system.graph.edgestatus; | ||
|
|
||
| import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; | ||
|
|
||
| import com.linkedin.datahub.upgrade.UpgradeContext; | ||
| import com.linkedin.datahub.upgrade.system.AbstractMCLStep; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.jetbrains.annotations.Nullable; | ||
|
|
||
| @Slf4j | ||
| public class ReindexReindexEdgeStatusStep extends AbstractMCLStep { | ||
|
|
||
| public ReindexReindexEdgeStatusStep( | ||
| OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| AspectDao aspectDao, | ||
| Integer batchSize, | ||
| Integer batchDelayMs, | ||
| Integer limit) { | ||
| super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit); | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return "edge-status-reindex-v1"; | ||
| } | ||
|
|
||
| @Nonnull | ||
| @Override | ||
| protected String getAspectName() { | ||
| return STATUS_ASPECT_NAME; | ||
| } | ||
|
|
||
| @Nullable | ||
| @Override | ||
| protected String getUrnLike() { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| /** | ||
| * Returns whether the upgrade should be skipped. Uses previous run history or the environment | ||
| * variable to determine whether to skip. | ||
| */ | ||
| public boolean skip(UpgradeContext context) { | ||
| boolean envFlagRecommendsSkip = Boolean.parseBoolean(System.getenv("SKIP_REINDEX_EDGE_STATUS")); | ||
| if (envFlagRecommendsSkip) { | ||
| log.info("Environment variable SKIP_REINDEX_EDGE_STATUS is set to true. Skipping."); | ||
| } | ||
| return (super.skip(context) || envFlagRecommendsSkip); | ||
| } | ||
| } |
2 changes: 1 addition & 1 deletion
2
...m/vianodes/ReindexDataJobViaNodesCLL.java → ...h/vianodes/ReindexDataJobViaNodesCLL.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
...anodes/ReindexDataJobViaNodesCLLStep.java → ...anodes/ReindexDataJobViaNodesCLLStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
8 changes: 8 additions & 0 deletions
8
entity-registry/src/main/java/com/linkedin/metadata/aspect/models/graph/EdgeUrnType.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| package com.linkedin.metadata.aspect.models.graph; | ||
|
|
||
| public enum EdgeUrnType { | ||
| SOURCE, | ||
| DESTINATION, | ||
| VIA, | ||
| LIFECYCLE_OWNER | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.