Skip to content

Commit

Permalink
feat(ui) Retrieve last ingested timestamp and display on frontend (#5600
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@chriscollins3456
chriscollins3456 authored Aug 12, 2022
1 parent 2252f82 commit 261163c
Show file tree
Hide file tree
Showing 26 changed files with 482 additions and 15 deletions.
4 changes: 4 additions & 0 deletions ...phql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -62,6 +63,9 @@ public Chart apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.CHART);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<Chart> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(CHART_KEY_ASPECT_NAME, this::mapChartKey);
mappingHelper.mapToResult(CHART_INFO_ASPECT_NAME, (entity, dataMap) -> this.mapChartInfo(entity, dataMap, entityUrn));
Expand Down
30 changes: 30 additions & 0 deletions ...main/java/com/linkedin/datahub/graphql/types/common/mappers/util/SystemMetadataUtils.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package com.linkedin.datahub.graphql.types.common.mappers.util;

import com.linkedin.entity.EnvelopedAspectMap;
import com.linkedin.mxe.SystemMetadata;

import javax.annotation.Nonnull;

import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;

public class SystemMetadataUtils {

private SystemMetadataUtils() {
}

public static Long getLastIngested(@Nonnull EnvelopedAspectMap aspectMap) {
Long lastIngested = null;
for (String aspect : aspectMap.keySet()) {
if (aspectMap.get(aspect).hasSystemMetadata()) {
SystemMetadata systemMetadata = aspectMap.get(aspect).getSystemMetadata();
if (systemMetadata.hasRunId() && !systemMetadata.getRunId().equals(DEFAULT_RUN_ID) && systemMetadata.hasLastObserved()) {
Long lastObserved = systemMetadata.getLastObserved();
if (lastIngested == null || lastObserved > lastIngested) {
lastIngested = lastObserved;
}
}
}
}
return lastIngested;
}
}
3 changes: 3 additions & 0 deletions ...e/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
Expand All @@ -39,6 +40,8 @@ public static Container map(final EntityResponse entityResponse) {
final Container result = new Container();
final Urn entityUrn = entityResponse.getUrn();
final EnvelopedAspectMap aspects = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspects);
result.setLastIngested(lastIngested);

result.setUrn(entityUrn.toString());
result.setType(EntityType.CONTAINER);
Expand Down
4 changes: 4 additions & 0 deletions ...e/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -59,6 +60,9 @@ public Dashboard apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.DASHBOARD);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<Dashboard> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(DASHBOARD_KEY_ASPECT_NAME, this::mapDashboardKey);
mappingHelper.mapToResult(DASHBOARD_INFO_ASPECT_NAME, (entity, dataMap) -> this.mapDashboardInfo(entity, dataMap, entityUrn));
Expand Down
4 changes: 4 additions & 0 deletions ...ore/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -54,6 +55,9 @@ public DataFlow apply(@Nonnull final EntityResponse entityResponse) {
Urn entityUrn = entityResponse.getUrn();

EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<DataFlow> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(DATA_FLOW_KEY_ASPECT_NAME, this::mapKey);
mappingHelper.mapToResult(DATA_FLOW_INFO_ASPECT_NAME, (entity, dataMap) -> this.mapInfo(entity, dataMap, entityUrn));
Expand Down
6 changes: 6 additions & 0 deletions ...-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,15 @@
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
import com.linkedin.datajob.EditableDataJobProperties;
import com.linkedin.domain.Domains;
import com.linkedin.entity.EntityResponse;
import com.linkedin.entity.EnvelopedAspectMap;
import com.linkedin.metadata.key.DataJobKey;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
Expand All @@ -54,6 +56,10 @@ public DataJob apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.DATA_JOB);

EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

entityResponse.getAspects().forEach((name, aspect) -> {
DataMap data = aspect.getValue().data();
if (DATA_JOB_KEY_ASPECT_NAME.equals(name)) {
Expand Down
4 changes: 4 additions & 0 deletions ...main/java/com/linkedin/datahub/graphql/types/dataplatform/mappers/DataPlatformMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.linkedin.datahub.graphql.generated.DataPlatform;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import com.linkedin.dataplatform.DataPlatformInfo;
import com.linkedin.entity.EntityResponse;
Expand Down Expand Up @@ -34,6 +35,9 @@ public DataPlatform apply(@Nonnull final EntityResponse entityResponse) {
result.setName(dataPlatformKey.getPlatformName());

EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<DataPlatform> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(DATA_PLATFORM_KEY_ASPECT_NAME, (dataPlatform, dataMap) ->
dataPlatform.setName(new DataPlatformKey(dataMap).getPlatformName()));
Expand Down
5 changes: 4 additions & 1 deletion ...-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -58,14 +59,16 @@ public static Dataset map(@Nonnull final EntityResponse dataset) {
return INSTANCE.apply(dataset);
}

@Override
public Dataset apply(@Nonnull final EntityResponse entityResponse) {
Dataset result = new Dataset();
Urn entityUrn = entityResponse.getUrn();
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.DATASET);

EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<Dataset> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(DATASET_KEY_ASPECT_NAME, this::mapDatasetKey);
mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, (entity, dataMap) -> this.mapDatasetProperties(entity, dataMap, entityUrn));
Expand Down
4 changes: 4 additions & 0 deletions ...ore/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -57,6 +58,9 @@ public MLFeature apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.MLFEATURE);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<MLFeature> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(ML_FEATURE_KEY_ASPECT_NAME, this::mapMLFeatureKey);
mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlFeature, dataMap) ->
Expand Down
4 changes: 4 additions & 0 deletions ...rc/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -56,6 +57,9 @@ public MLFeatureTable apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.MLFEATURE_TABLE);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<MLFeatureTable> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlFeatureTable, dataMap) ->
mlFeatureTable.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn)));
Expand Down
4 changes: 4 additions & 0 deletions .../src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -54,6 +55,9 @@ public MLModelGroup apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.MLMODEL_GROUP);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<MLModelGroup> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlModelGroup, dataMap) ->
mlModelGroup.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn)));
Expand Down
4 changes: 4 additions & 0 deletions ...-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -68,6 +69,9 @@ public MLModel apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.MLMODEL);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<MLModel> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(ML_MODEL_KEY_ASPECT_NAME, this::mapMLModelKey);
mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlModel, dataMap) ->
Expand Down
4 changes: 4 additions & 0 deletions .../src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -55,6 +56,9 @@ public MLPrimaryKey apply(@Nonnull final EntityResponse entityResponse) {
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.MLPRIMARY_KEY);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
result.setLastIngested(lastIngested);

MappingHelper<MLPrimaryKey> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlPrimaryKey, dataMap) ->
mlPrimaryKey.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn)));
Expand Down
Loading

0 comments on commit 261163c

Please sign in to comment.