Merge branch 'master' into urn-existence-validation-annotation

david-leifker · Feb 13, 2025 · 83b19cb · 83b19cb
2 parents c7df219 + 7472c53
commit 83b19cb
Show file tree

Hide file tree

Showing 137 changed files with 5,965 additions and 1,646 deletions.
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -27,7 +27,7 @@ concurrency:
 jobs:
   metadata-ingestion:
     runs-on: ubuntu-latest
-    timeout-minutes: 40
+    timeout-minutes: 60
     env:
       DATAHUB_TELEMETRY_ENABLED: false
       # TODO: Enable this once the test is fixed.

diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -51,7 +51,8 @@ jobs:
                 "acrylJonny",
                 "chakru-r",
                 "brock-acryl",
-                "mminichino"
+                "mminichino",
+                "jayacryl"
               ]'), 
               github.actor
             ) 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,4 +1,4 @@
-# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-17 16:43:31 UTC
+# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-02-11 10:00:11 UTC
 # Do not edit this file directly. Run the script to regenerate.
 # Add additional hooks in .github/scripts/pre-commit-override.yaml
 repos:
@@ -53,13 +53,6 @@ repos:
         files: ^metadata-dao-impl/kafka-producer/.*\.java$
         pass_filenames: false
 
-      - id: metadata-events-mxe-avro-spotless
-        name: metadata-events/mxe-avro Spotless Apply
-        entry: ./gradlew :metadata-events:mxe-avro:spotlessApply
-        language: system
-        files: ^metadata-events/mxe-avro/.*\.java$
-        pass_filenames: false
-
       - id: metadata-events-mxe-registration-spotless
         name: metadata-events/mxe-registration Spotless Apply
         entry: ./gradlew :metadata-events:mxe-registration:spotlessApply
@@ -312,6 +305,20 @@ repos:
         files: ^metadata-service/graphql-servlet-impl/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-iceberg-catalog-spotless
+        name: metadata-service/iceberg-catalog Spotless Apply
+        entry: ./gradlew :metadata-service:iceberg-catalog:spotlessApply
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-iceberg-catalog-lint-fix
+        name: metadata-service/iceberg-catalog Lint Fix
+        entry: ./gradlew :metadata-service:iceberg-catalog:lintFix
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.py$
+        pass_filenames: false
+
       - id: metadata-service-openapi-analytics-servlet-spotless
         name: metadata-service/openapi-analytics-servlet Spotless Apply
         entry: ./gradlew :metadata-service:openapi-analytics-servlet:spotlessApply
@@ -410,6 +417,13 @@ repos:
         files: ^metadata-service/servlet/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-war-spotless
+        name: metadata-service/war Spotless Apply
+        entry: ./gradlew :metadata-service:war:spotlessApply
+        language: system
+        files: ^metadata-service/war/.*\.java$
+        pass_filenames: false
+
       - id: metadata-utils-spotless
         name: metadata-utils Spotless Apply
         entry: ./gradlew :metadata-utils:spotlessApply

diff --git a/...l-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java b/...l-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java
@@ -48,6 +48,8 @@ public Schema apply(
     result.setPrimaryKeys(input.getPrimaryKeys());
     result.setFields(
         input.getFields().stream()
+            // do not map the field if the field.getFieldPath() is null or empty
+            .filter(field -> field.getFieldPath() != null && !field.getFieldPath().isEmpty())
             .map(field -> SchemaFieldMapper.map(context, field, entityUrn))
             .collect(Collectors.toList()));
     result.setPlatformSchema(PlatformSchemaMapper.map(context, input.getPlatformSchema()));

diff --git a/...rc/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java b/...rc/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java
@@ -46,6 +46,8 @@ public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(
     result.setPrimaryKeys(input.getPrimaryKeys());
     result.setFields(
         input.getFields().stream()
+            // do not map the field if the field.getFieldPath() is null or empty
+            .filter(field -> field.getFieldPath() != null && !field.getFieldPath().isEmpty())
             .map(field -> SchemaFieldMapper.map(context, field, entityUrn))
             .collect(Collectors.toList()));
     result.setPlatformSchema(PlatformSchemaMapper.map(context, input.getPlatformSchema()));

diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeUrnMapper.java b/...core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeUrnMapper.java
@@ -111,4 +111,8 @@ public static String getEntityTypeUrn(String name) {
     }
     return ENTITY_NAME_TO_ENTITY_TYPE_URN.get(name);
   }
+
+  public static boolean isValidEntityType(String entityTypeUrn) {
+    return ENTITY_TYPE_URN_TO_NAME.containsKey(entityTypeUrn);
+  }
 }
diff --git a/.../java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java b/.../java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java
@@ -20,6 +20,7 @@
 import com.linkedin.datahub.graphql.generated.StructuredPropertySettings;
 import com.linkedin.datahub.graphql.generated.TypeQualifier;
 import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
+import com.linkedin.datahub.graphql.types.entitytype.EntityTypeUrnMapper;
 import com.linkedin.datahub.graphql.types.mappers.MapperUtils;
 import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
 import com.linkedin.entity.EntityResponse;
@@ -30,7 +31,9 @@
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 public class StructuredPropertyMapper
     implements ModelMapper<EntityResponse, StructuredPropertyEntity> {
 
@@ -141,8 +144,21 @@ private TypeQualifier mapTypeQualifier(final StringArrayMap gmsTypeQualifier) {
     final TypeQualifier typeQualifier = new TypeQualifier();
     List<String> allowedTypes = gmsTypeQualifier.get(ALLOWED_TYPES);
     if (allowedTypes != null) {
+      // filter out correct allowedTypes
+      List<String> validAllowedTypes =
+          allowedTypes.stream()
+              .filter(EntityTypeUrnMapper::isValidEntityType)
+              .collect(Collectors.toList());
+      if (validAllowedTypes.size() != allowedTypes.size()) {
+        log.error(
+            String.format(
+                "Property has invalid allowed types set. Current list of allowed types: %s",
+                allowedTypes));
+      }
       typeQualifier.setAllowedTypes(
-          allowedTypes.stream().map(this::createEntityTypeEntity).collect(Collectors.toList()));
+          validAllowedTypes.stream()
+              .map(this::createEntityTypeEntity)
+              .collect(Collectors.toList()));
     }
     return typeQualifier;
   }

diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
@@ -95,7 +95,8 @@
         "uuid": "^8.3.2",
         "virtualizedtableforantd4": "^1.2.1",
         "web-vitals": "^0.2.4",
-        "yamljs": "^0.3.0"
+        "yamljs": "^0.3.0",
+        "country-data-list": "^1.3.4"
     },
     "scripts": {
         "analyze": "source-map-explorer 'dist/assets/*.js'",

diff --git a/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx b/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx
@@ -84,6 +84,38 @@ export function computeLines(chartData: TimeSeriesChartType, insertBlankPoints:
     return returnLines;
 }
 
+const formatAxisDate = (value: number, chartData: TimeSeriesChartType) => {
+    const date = new Date(value);
+
+    switch (chartData.interval) {
+        case 'MONTH':
+            return date.toLocaleDateString('en-US', {
+                month: 'short',
+                year: 'numeric',
+                timeZone: 'UTC',
+            });
+        case 'WEEK':
+            return date.toLocaleDateString('en-US', {
+                month: 'short',
+                day: 'numeric',
+                timeZone: 'UTC',
+            });
+        case 'DAY':
+            return date.toLocaleDateString('en-US', {
+                weekday: 'short',
+                day: 'numeric',
+                timeZone: 'UTC',
+            });
+        default:
+            return date.toLocaleDateString('en-US', {
+                month: 'short',
+                day: 'numeric',
+                year: 'numeric',
+                timeZone: 'UTC',
+            });
+    }
+};
+
 export const TimeSeriesChart = ({
     chartData,
     width,
@@ -117,6 +149,7 @@ export const TimeSeriesChart = ({
                     strokeWidth={style?.axisWidth}
                     tickLabelProps={{ fill: 'black', fontFamily: 'inherit', fontSize: 10 }}
                     numTicks={3}
+                    tickFormat={(value) => formatAxisDate(value, chartData)}
                 />
                 <Axis
                     orientation="right"
@@ -151,9 +184,7 @@ export const TimeSeriesChart = ({
                         tooltipData?.nearestDatum && (
                             <div>
                                 <div>
-                                    {new Date(
-                                        Number(accessors.xAccessor(tooltipData.nearestDatum.datum)),
-                                    ).toDateString()}
+                                    {formatAxisDate(accessors.xAccessor(tooltipData.nearestDatum.datum), chartData)}
                                 </div>
                                 <div>{accessors.yAccessor(tooltipData.nearestDatum.datum)}</div>
                             </div>

diff --git a/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx
@@ -183,9 +183,11 @@ export class DataProcessInstanceEntity implements Entity<DataProcessInstance> {
                 parentContainers={data.parentContainers}
                 parentEntities={parentEntities}
                 container={data.container || undefined}
-                duration={firstState?.durationMillis}
-                status={firstState?.result?.resultType}
-                startTime={firstState?.timestampMillis}
+                dataProcessInstanceProps={{
+                    startTime: firstState?.timestampMillis,
+                    duration: firstState?.durationMillis ?? undefined,
+                    status: firstState?.result?.resultType ?? undefined,
+                }}
             />
         );
     };

diff --git a/datahub-web-react/src/app/entity/dataProcessInstance/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataProcessInstance/preview/Preview.tsx
@@ -39,9 +39,7 @@ export const Preview = ({
     health,
     parentEntities,
     parentContainers,
-    duration,
-    status,
-    startTime,
+    dataProcessInstanceProps,
 }: {
     urn: string;
     name: string;
@@ -64,9 +62,11 @@ export const Preview = ({
     health?: Health[] | null;
     parentEntities?: Array<GeneratedEntity> | null;
     parentContainers?: ParentContainersResult | null;
-    duration?: number | null;
-    status?: string | null;
-    startTime?: number | null;
+    dataProcessInstanceProps?: {
+        startTime?: number;
+        duration?: number;
+        status?: string;
+    };
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
     return (
@@ -95,9 +95,7 @@ export const Preview = ({
             paths={paths}
             health={health || undefined}
             parentEntities={parentEntities}
-            duration={duration}
-            status={status}
-            startTime={startTime}
+            dataProcessInstanceProps={dataProcessInstanceProps}
         />
     );
 };
diff --git a/datahub-web-react/src/app/entity/shared/SidebarStyledComponents.tsx b/datahub-web-react/src/app/entity/shared/SidebarStyledComponents.tsx
@@ -64,6 +64,14 @@ export const Name = styled.div`
     margin: 13px 0 7px 0;
 `;
 
+export const UserDetails = styled.div`
+    font-size: 14px;
+    line-height: 35px;
+    font-weight: 400;
+    line-height: 28px;
+    text-align: center;
+`;
+
 export const TitleRole = styled.div`
     font-size: 14px;
     line-height: 22px;
@@ -74,6 +82,8 @@ export const TitleRole = styled.div`
 export const Team = styled.div`
     font-size: 12px;
     line-height: 20px;
+    font-weight: 400;
+    padding-bottom: 10px;
     color: #8c8c8c;
 `;
 
@@ -111,6 +121,23 @@ export const AboutSection = styled.div`
     color: #262626;
 `;
 
+export const LocationSection = styled.div`
+    text-align: left;
+    font-weight: bold;
+    font-size: 14px;
+    line-height: 26px;
+    color: #262626;
+`;
+
+export const LocationSectionText = styled.div`
+    text-align: left;
+    font-weight: normal;
+    font-size: 14px;
+    line-height: 26px;
+    margin-bottom: -10px;
+    color: #262626;
+`;
+
 export const AboutSectionText = styled.div`
     font-size: 12px;
     font-weight: 100;

diff --git a/...ub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx b/...ub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx
@@ -2,44 +2,20 @@ import React from 'react';
 import * as QueryString from 'query-string';
 import { useHistory, useLocation } from 'react-router';
 import { ApolloError } from '@apollo/client';
-import { EntityType, FacetFilterInput } from '../../../../../../types.generated';
+import { FacetFilterInput } from '../../../../../../types.generated';
 import useFilters from '../../../../../search/utils/useFilters';
 import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl';
 import { FilterSet, GetSearchResultsParams, SearchResultsInterface } from './types';
 import { useEntityQueryParams } from '../../../containers/profile/utils';
 import { EmbeddedListSearch } from './EmbeddedListSearch';
-import { UnionType } from '../../../../../search/utils/constants';
+import { EMBEDDED_LIST_SEARCH_ENTITY_TYPES, UnionType } from '../../../../../search/utils/constants';
 import {
     DownloadSearchResults,
     DownloadSearchResultsInput,
     DownloadSearchResultsParams,
 } from '../../../../../search/utils/types';
 
 const FILTER = 'filter';
-const SEARCH_ENTITY_TYPES = [
-    EntityType.Dataset,
-    EntityType.Dashboard,
-    EntityType.Chart,
-    EntityType.Mlmodel,
-    EntityType.MlmodelGroup,
-    EntityType.MlfeatureTable,
-    EntityType.Mlfeature,
-    EntityType.MlprimaryKey,
-    EntityType.DataFlow,
-    EntityType.DataJob,
-    EntityType.GlossaryTerm,
-    EntityType.GlossaryNode,
-    EntityType.Tag,
-    EntityType.Role,
-    EntityType.CorpUser,
-    EntityType.CorpGroup,
-    EntityType.Container,
-    EntityType.Domain,
-    EntityType.DataProduct,
-    EntityType.Notebook,
-    EntityType.BusinessAttribute,
-    EntityType.DataProcessInstance,
-];
 
 function getParamsWithoutFilters(params: QueryString.ParsedQuery<string>) {
     const paramsCopy = { ...params };
@@ -161,7 +137,7 @@ export const EmbeddedListSearchSection = ({
 
     return (
         <EmbeddedListSearch
-            entityTypes={SEARCH_ENTITY_TYPES}
+            entityTypes={EMBEDDED_LIST_SEARCH_ENTITY_TYPES}
             query={query || ''}
             page={page}
             unionType={unionType}

diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts
@@ -45,6 +45,7 @@ import {
     Documentation,
     DisplayProperties,
     VersionProperties,
+    DataProcessRunEvent,
 } from '../../../types.generated';
 import { FetchedEntity } from '../../lineage/types';
 
@@ -135,6 +136,9 @@ export type GenericEntityProperties = {
     displayProperties?: Maybe<DisplayProperties>;
     notes?: Maybe<EntityRelationshipsResult>;
     versionProperties?: Maybe<VersionProperties>;
+
+    // Data process instance
+    lastRunEvent?: Maybe<DataProcessRunEvent>;
 };
 
 export type GenericEntityUpdate = {