Merge branch 'master' into bigquery_audit_v2_support

.github/workflows/docker-ingestion.yml

@@ -70,6 +70,10 @@ jobs:
           # add git short SHA as Docker tag
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
         if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
@@ -81,6 +85,7 @@ jobs:
         with:
           context: .
           file: ./docker/datahub-ingestion/Dockerfile
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.docker_meta.outputs.tags }}
           push: ${{ needs.setup.outputs.publish == 'true' }}
           build-args: |

.github/workflows/metadata-ingestion.yml

@@ -19,6 +19,8 @@ jobs:
 
   metadata-ingestion-general:
     runs-on: ubuntu-latest
+    env:
+      SPARK_VERSION: 3.0.3
     strategy:
       matrix:
         python-version: ["3.6", "3.9.9"]
@@ -46,6 +48,8 @@ jobs:
 
   metadata-ingestion-by-version:
     runs-on: ubuntu-latest
+    env:
+      SPARK_VERSION: 3.0.3
     strategy:
       matrix:
         python-version: ["3.6", "3.9.9"]

.gitignore

@@ -57,3 +57,6 @@ smoke-test/spark-smoke-test/__pycache__/
 
 # Metadata Ingestion Generated
 metadata-ingestion/generated/**
+
+# docs
+docs/generated/

README.md

@@ -49,10 +49,10 @@ HOSTED_DOCS_ONLY-->
 [Town Hall](https://datahubproject.io/docs/townhalls)
 
 ---
-> 📣 Next DataHub town hall meeting on Dec 17th, 9am-10am PDT ([convert to your local time](https://greenwichmeantime.com/time/to/pacific-local/))
+> 📣 Next DataHub town hall meeting on Feb 25th, 9am-10am PDT ([convert to your local time](https://greenwichmeantime.com/time/to/pacific-local/))
 >
 > - Topic Proposals: [submit here](https://docs.google.com/forms/d/1v2ynbAXjJlqY97xE_X1DAntNrXDznOFiNfryUkMPtkI/)
-> - Signup to get a calendar invite: [here](https://www.addevent.com/event/WE11214120)
+> - Signup to get a calendar invite: [here](https://www.addevent.com/event/lD12111844)
 > - Town-hall Zoom link: [zoom.datahubproject.io](https://zoom.datahubproject.io)
 > - [Meeting details](docs/townhalls.md) & [past recordings](docs/townhall-history.md)
 
@@ -107,6 +107,7 @@ Join our [slack workspace](https://slack.datahubproject.io) for discussions and
 
 Here are the companies that have officially adopted DataHub. Please feel free to add yours to the list if we missed it.
 
+- [Adevinta](https://www.adevinta.com/)
 - [Banksalad](https://www.banksalad.com)
 - [DefinedCrowd](http://www.definedcrowd.com)
 - [DFDS](https://www.dfds.com/)

build.gradle

@@ -120,15 +120,18 @@ project.ext.externalDependency = [
     'springBeans': 'org.springframework:spring-beans:5.2.3.RELEASE',
     'springContext': 'org.springframework:spring-context:5.2.3.RELEASE',
     'springCore': 'org.springframework:spring-core:5.2.3.RELEASE',
+    'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.6',
     'springJdbc': 'org.springframework:spring-jdbc:5.2.3.RELEASE',
     'springWeb': 'org.springframework:spring-web:5.2.3.RELEASE',
     'springWebMVC': 'org.springframework:spring-webmvc:5.2.3.RELEASE',
+    'springBoot': 'org.springframework.boot:spring-boot:2.1.14.RELEASE',
     'springBootAutoconfigure': 'org.springframework.boot:spring-boot-autoconfigure:2.1.4.RELEASE',
     'springBootStarterWeb': 'org.springframework.boot:spring-boot-starter-web:2.1.4.RELEASE',
     'springBootStarterJetty': 'org.springframework.boot:spring-boot-starter-jetty:2.1.4.RELEASE',
     'springBootStarterCache': 'org.springframework.boot:spring-boot-starter-cache:2.1.4.RELEASE',
     'springKafka': 'org.springframework.kafka:spring-kafka:2.2.14.RELEASE',
     'springActuator': 'org.springframework.boot:spring-boot-starter-actuator:2.1.4.RELEASE',
+    'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12',
     'testng': 'org.testng:testng:7.3.0',
     'testContainers': 'org.testcontainers:testcontainers:1.15.3',
     'testContainersJunit': 'org.testcontainers:junit-jupiter:1.15.3',

datahub-frontend/conf/routes

@@ -24,6 +24,12 @@ POST          /api/*path                                                      co
 DELETE        /api/*path                                                      controllers.Application.proxy(path)
 PUT           /api/*path                                                      controllers.Application.proxy(path)
 
+# Proxies API requests to the metadata service api
+GET           /openapi/*path                                                  controllers.Application.proxy(path)
+POST          /openapi/*path                                                  controllers.Application.proxy(path)
+DELETE        /openapi/*path                                                  controllers.Application.proxy(path)
+PUT           /openapi/*path                                                  controllers.Application.proxy(path)
+
 # Map static resources from the /public folder to the /assets URL path
 GET           /assets/*file                                                   controllers.Assets.at(path="/public", file)
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -8,6 +8,7 @@
 import com.linkedin.datahub.graphql.analytics.resolver.GetMetadataAnalyticsResolver;
 import com.linkedin.datahub.graphql.analytics.resolver.IsAnalyticsEnabledResolver;
 import com.linkedin.datahub.graphql.analytics.service.AnalyticsService;
+import com.linkedin.datahub.graphql.generated.ActorFilter;
 import com.linkedin.datahub.graphql.generated.AggregationMetadata;
 import com.linkedin.datahub.graphql.generated.Aspect;
 import com.linkedin.datahub.graphql.generated.BrowseResults;
@@ -455,6 +456,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
         configureContainerResolvers(builder);
         configureGlossaryTermResolvers(builder);
         configureDomainResolvers(builder);
+        configurePolicyResolvers(builder);
     }
 
     public GraphQLEngine.Builder builder() {
@@ -1112,6 +1114,24 @@ private void configureDomainResolvers(final RuntimeWiring.Builder builder) {
         );
     }
 
+    private void configurePolicyResolvers(final RuntimeWiring.Builder builder) {
+        // Register resolvers for "resolvedUsers" and "resolvedGroups" field of the Policy type.
+        builder.type("ActorFilter", typeWiring -> typeWiring
+            .dataFetcher("resolvedUsers", new LoadableTypeBatchResolver<>(corpUserType,
+                (env) -> {
+                    final ActorFilter filter = env.getSource();
+                    return filter.getUsers();
+                }
+            ))
+            .dataFetcher("resolvedGroups", new LoadableTypeBatchResolver<>(corpGroupType,
+                    (env) -> {
+                        final ActorFilter filter = env.getSource();
+                        return filter.getGroups();
+                    }
+            ))
+        );
+    }
+
     private <T> DataLoader<String, DataFetcherResult<T>> createDataLoader(final LoadableType<T> graphType, final QueryContext queryContext) {
         BatchLoaderContextProvider contextProvider = () -> queryContext;
         DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/LoadableTypeBatchResolver.java

@@ -33,6 +33,9 @@ public LoadableTypeBatchResolver(final LoadableType<T> loadableType, final Funct
     @Override
     public CompletableFuture<List<T>> get(DataFetchingEnvironment environment) {
         final List<String> urns = _urnProvider.apply(environment);
+        if (urns == null) {
+            return null;
+        }
         final DataLoader<String, T> loader = environment.getDataLoaderRegistry().getDataLoader(_loadableType.name());
         return loader.loadMany(urns);
     }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -4724,6 +4724,16 @@ type ActorFilter {
   Whether the filter should apply to all groups
   """
   allGroups: Boolean!
+
+  """
+  The list of users on the Policy, resolved.
+  """
+  resolvedUsers: [CorpUser!]
+
+  """
+  The list of groups on the Policy, resolved.
+  """
+  resolvedGroups: [CorpGroup!]
 }
 
 """

datahub-graphql-core/src/main/resources/ingestion.graphql

@@ -19,13 +19,13 @@ extend type Query {
 
   """
   Fetch a specific ingestion source
-  urn: The primary key associated with the ingestion execution request.
+  urn: The primary key associated with the ingestion source.
   """
   ingestionSource(urn: String!): IngestionSource
 
   """
-  Create a request to execute an ingestion job
-  urn: The primary key associated with the ingestion execution request.
+  Get an execution request
+  urn: The primary key associated with the execution request.
   """
   executionRequest(urn: String!): ExecutionRequest
 }

datahub-web-react/package.json

@@ -68,6 +68,7 @@
         "query-string": "^6.13.8",
         "rc-table": "^7.13.1",
         "react": "^17.0.0",
+        "react-color": "^2.19.3",
         "react-dom": "^17.0.0",
         "react-icons": "^4.2.0",
         "react-router": "^5.2.0",

datahub-web-react/src/Mocks.tsx

@@ -57,6 +57,11 @@ const user1 = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -87,6 +92,11 @@ const user2 = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -324,6 +334,11 @@ export const dataset3 = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -880,6 +895,11 @@ const sampleTag = {
             time: 0,
         },
     },
+    properties: {
+        name: 'abc-sample-tag',
+        description: 'sample tag description',
+        colorHex: 'sample tag color',
+    },
 };
 
 export const dataFlow1 = {
@@ -924,6 +944,11 @@ export const dataFlow1 = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -981,6 +1006,11 @@ export const dataJob1 = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -1049,6 +1079,11 @@ export const dataJob2 = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -1103,6 +1138,11 @@ export const dataJob3 = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -1169,6 +1209,11 @@ export const mlModel = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],
@@ -1224,6 +1269,11 @@ export const mlModelGroup = {
                     urn: 'urn:li:tag:abc-sample-tag',
                     name: 'abc-sample-tag',
                     description: 'sample tag',
+                    properties: {
+                        name: 'abc-sample-tag',
+                        description: 'sample tag',
+                        colorHex: 'sample tag color',
+                    },
                 },
             },
         ],

datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx

@@ -24,7 +24,7 @@ type Props = {
     description?: string | undefined;
     original?: string | undefined;
     onClose: () => void;
-    onSubmit: (description: string | null) => void;
+    onSubmit: (description: string) => void;
     isAddDesc?: boolean;
 };
 

datahub-web-react/src/app/entity/shared/components/styled/StyledTag.tsx

@@ -6,7 +6,7 @@ const generateColor = new ColorHash({
     saturation: 0.9,
 });
 
-export const StyledTag = styled(Tag)<{ $colorHash?: string }>`
+export const StyledTag = styled(Tag)<{ $color: any; $colorHash?: string }>`
     ${(props) =>
         props.$colorHash &&
         css`
@@ -15,7 +15,7 @@ export const StyledTag = styled(Tag)<{ $colorHash?: string }>`
                 content: '';
                 width: 8px;
                 height: 8px;
-                background: ${generateColor.hex(props.$colorHash)};
+                background: ${props.$color === null ? generateColor.hex(props.$colorHash) : props.$color};
                 border-radius: 100em;
                 margin-right: 3px;
             }

datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/AddOwnerModal.tsx

@@ -6,7 +6,6 @@ import { useAddOwnerMutation } from '../../../../../../../graphql/mutations.gene
 import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/search.generated';
 import { CorpUser, EntityType, OwnerEntityType, SearchResult } from '../../../../../../../types.generated';
 import { useEntityRegistry } from '../../../../../../useEntityRegistry';
-import { useEntityData } from '../../../../EntityContext';
 import { CustomAvatar } from '../../../../../../shared/avatar';
 import analytics, { EventType, EntityActionType } from '../../../../../../analytics';
 import { useEnterKeyListener } from '../../../../../../shared/useEnterKeyListener';
@@ -15,6 +14,8 @@ type Props = {
     visible: boolean;
     onClose: () => void;
     refetch?: () => Promise<any>;
+    urn: string;
+    entityType: EntityType;
 };
 
 const SearchResultContainer = styled.div`
@@ -40,9 +41,8 @@ type SelectedActor = {
     urn: string;
 };
 
-export const AddOwnerModal = ({ visible, onClose, refetch }: Props) => {
+export const AddOwnerModal = ({ visible, onClose, refetch, urn, entityType }: Props) => {
     const entityRegistry = useEntityRegistry();
-    const { urn, entityType } = useEntityData();
     const [selectedActor, setSelectedActor] = useState<SelectedActor | undefined>(undefined);
     const [userSearch, { data: userSearchData }] = useGetSearchResultsLazyQuery();
     const [groupSearch, { data: groupSearchData }] = useGetSearchResultsLazyQuery();

datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/SidebarOwnerSection.tsx

@@ -8,7 +8,7 @@ import { SidebarHeader } from '../SidebarHeader';
 import { AddOwnerModal } from './AddOwnerModal';
 
 export const SidebarOwnerSection = () => {
-    const { urn, entityData } = useEntityData();
+    const { urn, entityData, entityType } = useEntityData();
     const refetch = useRefetch();
     const [showAddModal, setShowAddModal] = useState(false);
     const ownersEmpty = !entityData?.ownership?.owners?.length;
@@ -36,6 +36,8 @@ export const SidebarOwnerSection = () => {
                 onClose={() => {
                     setShowAddModal(false);
                 }}
+                urn={urn}
+                entityType={entityType}
             />
         </div>
     );