From 5926c2278053a1fa62f0a8866f911c52bdf8b409 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Mon, 2 Aug 2021 22:04:12 -0700 Subject: [PATCH 1/7] Code Cleanup Part 1: Removing Dao Module & endpoints in DataHub Frontend --- datahub-dao/build.gradle | 15 - .../exceptions/MissingFieldException.java | 11 - .../com/linkedin/datahub/dao/DaoFactory.java | 117 ------- .../datahub/dao/table/DataPlatformsDao.java | 29 -- .../datahub/dao/table/DatasetOwnerDao.java | 30 -- .../datahub/dao/table/DatasetsDao.java | 21 -- .../linkedin/datahub/dao/table/GmsDao.java | 47 --- .../datahub/dao/table/LineageDao.java | 57 ---- .../linkedin/datahub/dao/view/BrowseDAO.java | 67 ---- .../datahub/dao/view/CorpUserViewDao.java | 63 ---- .../datahub/dao/view/DatasetViewDao.java | 144 --------- .../datahub/dao/view/DocumentSearchDao.java | 54 ---- .../datahub/dao/view/OwnerViewDao.java | 78 ----- .../datahub/models/PagedCollection.java | 23 -- .../datahub/models/table/CompanyUser.java | 25 -- .../datahub/models/table/Dataset.java | 40 --- .../linkedin/datahub/models/table/Group.java | 23 -- .../linkedin/datahub/models/table/User.java | 28 -- .../datahub/models/table/UserEntity.java | 25 -- .../datahub/models/table/UserSetting.java | 24 -- .../datahub/models/view/DataOriginView.java | 24 -- .../datahub/models/view/DatasetColumn.java | 46 --- .../datahub/models/view/DatasetOwner.java | 44 --- .../datahub/models/view/DatasetOwnership.java | 23 -- .../datahub/models/view/DatasetSchema.java | 23 -- .../datahub/models/view/DatasetView.java | 41 --- .../datahub/models/view/LineageView.java | 16 - .../linkedin/datahub/util/CorpUserUtil.java | 78 ----- .../linkedin/datahub/util/DatasetUtil.java | 101 ------- .../com/linkedin/datahub/util/OwnerUtil.java | 78 ----- .../com/linkedin/datahub/util/RestliUtil.java | 149 --------- .../com/linkedin/datahub/util/Search.java | 30 -- .../com/linkedin/datahub/util/UrnUtil.java | 45 --- .../main/resources/META-INF/persistence.xml | 12 - .../datahub/dao/view/CorpUserViewDaoTest.java | 83 ----- .../linkedin/datahub/util/RestliUtilTest.java | 54 ---- .../app/controllers/Application.java | 102 ------- .../app/controllers/api/v1/Dataset.java | 32 -- .../app/controllers/api/v1/User.java | 118 -------- .../app/controllers/api/v2/Browse.java | 104 ------- .../app/controllers/api/v2/CorpUser.java | 87 ------ .../app/controllers/api/v2/Dataset.java | 286 ------------------ .../app/controllers/api/v2/Search.java | 135 --------- .../app/utils/ControllerUtil.java | 58 ---- datahub-frontend/conf/routes | 29 -- datahub-frontend/play.gradle | 1 - 46 files changed, 2720 deletions(-) delete mode 100644 datahub-dao/build.gradle delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/common/exceptions/MissingFieldException.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/DaoFactory.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DataPlatformsDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetOwnerDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetsDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/table/GmsDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/table/LineageDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/view/BrowseDAO.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/view/CorpUserViewDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DatasetViewDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DocumentSearchDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/dao/view/OwnerViewDao.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/PagedCollection.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/table/CompanyUser.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/table/Dataset.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/table/Group.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/table/User.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserEntity.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserSetting.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/view/DataOriginView.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetColumn.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwner.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwnership.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetSchema.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetView.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/models/view/LineageView.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/util/CorpUserUtil.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/util/DatasetUtil.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/util/OwnerUtil.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/util/RestliUtil.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/util/Search.java delete mode 100644 datahub-dao/src/main/java/com/linkedin/datahub/util/UrnUtil.java delete mode 100644 datahub-dao/src/main/resources/META-INF/persistence.xml delete mode 100644 datahub-dao/src/test/java/com/linkedin/datahub/dao/view/CorpUserViewDaoTest.java delete mode 100644 datahub-dao/src/test/java/com/linkedin/datahub/util/RestliUtilTest.java delete mode 100644 datahub-frontend/app/controllers/api/v1/Dataset.java delete mode 100644 datahub-frontend/app/controllers/api/v1/User.java delete mode 100644 datahub-frontend/app/controllers/api/v2/Browse.java delete mode 100644 datahub-frontend/app/controllers/api/v2/CorpUser.java delete mode 100644 datahub-frontend/app/controllers/api/v2/Dataset.java delete mode 100644 datahub-frontend/app/controllers/api/v2/Search.java delete mode 100644 datahub-frontend/app/utils/ControllerUtil.java diff --git a/datahub-dao/build.gradle b/datahub-dao/build.gradle deleted file mode 100644 index e79c8c2b3c5173..00000000000000 --- a/datahub-dao/build.gradle +++ /dev/null @@ -1,15 +0,0 @@ -apply plugin: 'java' - -dependencies { - compile project(':gms:client') - compile project(':metadata-utils') - compile externalDependency.elasticSearchRest - compile externalDependency.hibernateCore - compile externalDependency.guava - - compileOnly externalDependency.lombok - - annotationProcessor externalDependency.lombok - - testCompile externalDependency.mockito -} \ No newline at end of file diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/common/exceptions/MissingFieldException.java b/datahub-dao/src/main/java/com/linkedin/datahub/common/exceptions/MissingFieldException.java deleted file mode 100644 index 8ecc578d1833a5..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/common/exceptions/MissingFieldException.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.linkedin.datahub.common.exceptions; - -/** - * An exception to be thrown when certain required field is missing. - */ -public class MissingFieldException extends RuntimeException { - - public MissingFieldException(String message) { - super(message); - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/DaoFactory.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/DaoFactory.java deleted file mode 100644 index 50346c2a1d8806..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/DaoFactory.java +++ /dev/null @@ -1,117 +0,0 @@ -package com.linkedin.datahub.dao; - -import com.linkedin.datahub.dao.table.DataPlatformsDao; -import com.linkedin.datahub.dao.table.DatasetOwnerDao; -import com.linkedin.datahub.dao.table.DatasetsDao; -import com.linkedin.datahub.dao.table.GmsDao; -import com.linkedin.datahub.dao.table.LineageDao; -import com.linkedin.datahub.dao.view.BrowseDAO; -import com.linkedin.datahub.dao.view.CorpUserViewDao; -import com.linkedin.datahub.dao.view.DatasetViewDao; -import com.linkedin.datahub.dao.view.DocumentSearchDao; -import com.linkedin.datahub.dao.view.OwnerViewDao; -import com.linkedin.util.Configuration; - -public class DaoFactory { - - private static final String GMS_HOST_ENV_VAR = "DATAHUB_GMS_HOST"; - private static final String GMS_PORT_ENV_VAR = "DATAHUB_GMS_PORT"; - private static final String GMS_USE_SSL_ENV_VAR = "DATAHUB_GMS_USE_SSL"; - private static final String GMS_SSL_PROTOCOL_VAR = "DATAHUB_GMS_SSL_PROTOCOL"; - - private static GmsDao _gmsDao; - private static DocumentSearchDao datasetDocumentSearchDao; - private static DocumentSearchDao corpUserDocumentSearchDao; - private static CorpUserViewDao corpUserViewDao; - private static BrowseDAO datasetBrowseDao; - private static OwnerViewDao ownerViewDao; - private static DatasetViewDao datasetViewDao; - private static DatasetOwnerDao datasetOwnerDao; - private static DatasetsDao datasetsDao; - private static LineageDao lineageDao; - private static DataPlatformsDao dataPlatformsDao; - - private DaoFactory() { - } - - private static GmsDao getGmsDao() { - if (_gmsDao == null) { - _gmsDao = new GmsDao(Configuration.getEnvironmentVariable(GMS_HOST_ENV_VAR), - Integer.parseInt(Configuration.getEnvironmentVariable(GMS_PORT_ENV_VAR)), - Boolean.parseBoolean(Configuration.getEnvironmentVariable(GMS_USE_SSL_ENV_VAR, "False")), - Configuration.getEnvironmentVariable(GMS_SSL_PROTOCOL_VAR)); - } - return _gmsDao; - } - - public static DocumentSearchDao getDatasetDocumentSearchDao() { - if (datasetDocumentSearchDao == null) { - datasetDocumentSearchDao = new DocumentSearchDao<>(getGmsDao().get_datasets()); - } - return datasetDocumentSearchDao; - } - - public static DocumentSearchDao getCorpUserDocumentSearchDao() { - if (corpUserDocumentSearchDao == null) { - corpUserDocumentSearchDao = new DocumentSearchDao<>(getGmsDao().get_corpUsers()); - } - return corpUserDocumentSearchDao; - } - - public static BrowseDAO getDatasetBrowseDAO() { - if (datasetBrowseDao == null) { - datasetBrowseDao = new BrowseDAO<>(getGmsDao().get_datasets()); - } - return datasetBrowseDao; - } - - public static CorpUserViewDao getCorpUserViewDao() { - if (corpUserViewDao == null) { - corpUserViewDao = new CorpUserViewDao(getGmsDao().get_corpUsers()); - } - return corpUserViewDao; - } - - public static OwnerViewDao getOwnerViewDao() { - if (ownerViewDao == null) { - ownerViewDao = new OwnerViewDao(getGmsDao().get_ownerships(), getGmsDao().get_corpUsers()); - } - return ownerViewDao; - } - - public static DatasetViewDao getDatasetViewDao() { - if (datasetViewDao == null) { - datasetViewDao = new DatasetViewDao(getGmsDao().get_datasets(), getGmsDao().get_deprecations(), - getGmsDao().get_institutionalMemory(), getGmsDao().get_schemas()); - } - return datasetViewDao; - } - - public static DatasetOwnerDao getDatasetOwnerDao() { - if (datasetOwnerDao == null) { - datasetOwnerDao = new DatasetOwnerDao(getGmsDao().get_ownerships()); - } - return datasetOwnerDao; - } - - public static DatasetsDao getDatasetsDao() { - if (datasetsDao == null) { - datasetsDao = new DatasetsDao(getGmsDao().get_ownerships()); - } - return datasetsDao; - } - - public static LineageDao getLineageDao() { - if (lineageDao == null) { - lineageDao = new LineageDao(getGmsDao().get_lineages(), getGmsDao().get_datasets()); - } - return lineageDao; - } - - public static DataPlatformsDao getDataPlatformsDao() { - if (dataPlatformsDao == null) { - dataPlatformsDao = new DataPlatformsDao(getGmsDao().get_dataPlatforms()); - } - return dataPlatformsDao; - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DataPlatformsDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DataPlatformsDao.java deleted file mode 100644 index 1e87735b404b98..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DataPlatformsDao.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.linkedin.datahub.dao.table; - -import com.linkedin.dataPlatforms.DataPlatform; -import com.linkedin.dataplatform.client.DataPlatforms; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; - - -public class DataPlatformsDao { - - private final DataPlatforms _dataPlatforms; - - public DataPlatformsDao(@Nonnull DataPlatforms dataPlatforms) { - _dataPlatforms = dataPlatforms; - } - - /** - * Get all data platforms - */ - public List> getAllPlatforms() throws Exception { - return _dataPlatforms.getAllPlatforms() - .stream() - .filter(DataPlatform::hasDataPlatformInfo) - .map(platform -> platform.getDataPlatformInfo().data()) - .collect(Collectors.toList()); - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetOwnerDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetOwnerDao.java deleted file mode 100644 index 5aea5941011f11..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetOwnerDao.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.linkedin.datahub.dao.table; - -import com.linkedin.common.OwnerArray; -import com.linkedin.common.Ownership; -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.datahub.models.view.DatasetOwner; -import com.linkedin.dataset.client.Ownerships; -import java.util.List; -import javax.annotation.Nonnull; - -import static com.linkedin.datahub.util.DatasetUtil.*; -import static com.linkedin.datahub.util.OwnerUtil.*; - -public class DatasetOwnerDao { - private final Ownerships _ownerships; - - public DatasetOwnerDao(@Nonnull Ownerships ownerships) { - this._ownerships = ownerships; - } - - public void updateDatasetOwners(@Nonnull String datasetUrn, @Nonnull List owners, - @Nonnull String user) throws Exception { - OwnerArray ownerArray = new OwnerArray(); - for (DatasetOwner owner : owners) { - ownerArray.add(toTmsOwner(owner)); - } - _ownerships.createOwnership(toDatasetUrn(datasetUrn), new Ownership().setOwners(ownerArray), - new CorpuserUrn(user)); - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetsDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetsDao.java deleted file mode 100644 index 47c878f1030f74..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetsDao.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.linkedin.datahub.dao.table; - -import com.linkedin.dataset.client.Ownerships; - -import javax.annotation.Nonnull; -import java.util.Arrays; -import java.util.List; - - -public class DatasetsDao { - - private final Ownerships _ownership; - - public DatasetsDao(@Nonnull Ownerships ownerships) { - this._ownership = ownerships; - } - - public List getDatasetOwnerTypes() { - return Arrays.asList("DataOwner", "Producer", "Delegate", "Stakeholder", "Consumer", "Developer"); - } -} \ No newline at end of file diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/GmsDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/GmsDao.java deleted file mode 100644 index 5fa4493fb29536..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/GmsDao.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.linkedin.datahub.dao.table; - -import com.linkedin.dataplatform.client.DataPlatforms; -import com.linkedin.dataset.client.Datasets; -import com.linkedin.dataset.client.Deprecations; -import com.linkedin.dataset.client.InstitutionalMemory; -import com.linkedin.dataset.client.Lineages; -import com.linkedin.dataset.client.Ownerships; -import com.linkedin.dataset.client.Schemas; -import com.linkedin.identity.client.CorpUsers; -import com.linkedin.metadata.restli.DefaultRestliClientFactory; -import com.linkedin.restli.client.Client; -import javax.annotation.Nonnull; -import lombok.Getter; - - -@Getter -public class GmsDao { - - private final CorpUsers _corpUsers; - private final Datasets _datasets; - private final Ownerships _ownerships; - private final InstitutionalMemory _institutionalMemory; - private final Deprecations _deprecations; - private final Schemas _schemas; - private final Lineages _lineages; - private final DataPlatforms _dataPlatforms; - - public GmsDao(@Nonnull Client restClient) { - _corpUsers = new CorpUsers(restClient); - _datasets = new Datasets(restClient); - _ownerships = new Ownerships(restClient); - _institutionalMemory = new InstitutionalMemory(restClient); - _deprecations = new Deprecations(restClient); - _schemas = new Schemas(restClient); - _lineages = new Lineages(restClient); - _dataPlatforms = new DataPlatforms(restClient); - } - - public GmsDao(@Nonnull String restliHostName, @Nonnull int restliHostPort) { - this(DefaultRestliClientFactory.getRestLiClient(restliHostName, restliHostPort)); - } - - public GmsDao(@Nonnull String restliHostName, @Nonnull int restliHostPort, boolean useSSL, String sslProtocol) { - this(DefaultRestliClientFactory.getRestLiClient(restliHostName, restliHostPort, useSSL, sslProtocol)); - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/LineageDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/LineageDao.java deleted file mode 100644 index 9133d6c28c48df..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/table/LineageDao.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.linkedin.datahub.dao.table; - -import com.linkedin.common.urn.DatasetUrn; -import com.linkedin.datahub.models.view.LineageView; -import com.linkedin.dataset.Dataset; -import com.linkedin.dataset.DownstreamArray; -import com.linkedin.dataset.UpstreamArray; -import com.linkedin.dataset.client.Datasets; -import com.linkedin.dataset.client.Lineages; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; - -import static com.linkedin.datahub.util.DatasetUtil.*; - - -public class LineageDao { - - private final Lineages _lineages; - private final Datasets _datasets; - - public LineageDao(@Nonnull Lineages lineages, @Nonnull Datasets datasets) { - _lineages = lineages; - _datasets = datasets; - } - - /** - * Gets the upstream datasets of a certain dataset with lineage metadata attached - * @param datasetUrn String - * @return List of LineageView - */ - public List getUpstreamLineage(@Nonnull String datasetUrn) throws Exception { - final UpstreamArray upstreamArray = _lineages.getUpstreamLineage(toDatasetUrn(datasetUrn)).getUpstreams(); - final Map datasets = _datasets.batchGet(upstreamArray.stream().map(u -> u.getDataset()) - .collect(Collectors.toSet())); - - return upstreamArray.stream() - .map(us -> toLineageView(datasets.get(us.getDataset()), us.getType().name(), us.getAuditStamp())) - .collect(Collectors.toList()); - } - - /** - * Gets the downstream datasets of a certain dataset with lineage metadata attached - * @param datasetUrn String - * @return List of LineageView - */ - public List getDownstreamLineage(@Nonnull String datasetUrn) throws Exception { - final DownstreamArray downstreamArray = _lineages.getDownstreamLineage(toDatasetUrn(datasetUrn)).getDownstreams(); - final Map datasets = _datasets.batchGet(downstreamArray.stream().map(u -> u.getDataset()) - .collect(Collectors.toSet())); - - return downstreamArray.stream() - .map(ds -> toLineageView(datasets.get(ds.getDataset()), ds.getType().name(), ds.getAuditStamp())) - .collect(Collectors.toList()); - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/BrowseDAO.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/BrowseDAO.java deleted file mode 100644 index 6601bc2de2c28b..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/BrowseDAO.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.linkedin.datahub.dao.view; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.linkedin.chart.client.Charts; -import com.linkedin.dashboard.client.Dashboards; -import com.linkedin.data.template.StringArray; -import com.linkedin.datahub.util.DatasetUtil; -import com.linkedin.dataset.client.Datasets; -import com.linkedin.metadata.query.BrowseResult; -import com.linkedin.metadata.query.BrowseResultEntityArray; -import com.linkedin.r2.RemoteInvocationException; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.Map; -import javax.annotation.Nonnull; - -import static com.linkedin.datahub.util.RestliUtil.*; -import static java.util.stream.Collectors.toList; - - -public class BrowseDAO { - private final CLIENT _client; - - public BrowseDAO(@Nonnull CLIENT client) { - this._client = client; - } - - @Nonnull - public JsonNode browse(@Nonnull String path, @Nonnull Map requestFilters, - int start, int count) throws RemoteInvocationException, IOException { - final BrowseResult resp; - if (_client instanceof Datasets) { - resp = ((Datasets) _client).browse(path, requestFilters, start, count); - } else if (_client instanceof Dashboards) { - resp = ((Dashboards) _client).browse(path, requestFilters, start, count); - } else if (_client instanceof Charts) { - resp = ((Charts) _client).browse(path, requestFilters, start, count); - } else { - throw new IllegalArgumentException("Unexpected client type: " + _client.getClass().getName()); - } - return getJsonFromBrowseResult(resp); - } - - @Nonnull - public JsonNode getBrowsePaths(@Nonnull String urn) throws RemoteInvocationException, URISyntaxException { - final StringArray response; - if (_client instanceof Datasets) { - response = ((Datasets) _client).getBrowsePaths(DatasetUtil.toDatasetUrn(urn)); - } else { - throw new IllegalArgumentException("Unexpected client type: " + _client.getClass().getName()); - } - return stringCollectionToArrayNode(response.stream().collect(toList())); - } - - @Nonnull - public JsonNode getJsonFromBrowseResult(@Nonnull BrowseResult browseResult) throws IOException { - final ObjectNode node = OM.createObjectNode(); - final BrowseResultEntityArray browseResultEntityArray = browseResult.getEntities(); - node.set("elements", collectionToArrayNode(browseResultEntityArray.subList(0, browseResultEntityArray.size()))); - node.put("start", browseResult.getFrom()); - node.put("count", browseResult.getPageSize()); - node.put("total", browseResult.getNumEntities()); - node.set("metadata", toJsonNode(browseResult.getMetadata())); - return node; - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/CorpUserViewDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/CorpUserViewDao.java deleted file mode 100644 index 616baa403aaff5..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/CorpUserViewDao.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.linkedin.datahub.dao.view; - -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.datahub.util.CorpUserUtil; -import com.linkedin.identity.CorpUser; -import com.linkedin.identity.CorpUserEditableInfo; -import com.linkedin.identity.client.CorpUsers; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import java.util.List; - - -@Slf4j -public class CorpUserViewDao { - - private static final String MANAGER_FIELD_NAME = "managerName"; - - private final CorpUsers _corpUsers; - - public CorpUserViewDao(@Nonnull CorpUsers corpUsers) { - this._corpUsers = corpUsers; - } - - /** - * Adds manager name to {@link CorpUser} if managerUrn is present in info - * @param corpUserUrn corpUser urn string - * @return JSON node corresponding to the {@link CorpUser} - * @throws Exception - */ - @Nonnull - public CorpUser get(@Nonnull String corpUserUrn) throws Exception { - CorpUser corpUser = _corpUsers.get(CorpUserUtil.toCorpUserUrn(corpUserUrn)); - if (corpUser.getInfo() == null) { - return corpUser; - } - CorpuserUrn managerUrn = corpUser.getInfo().getManagerUrn(); - if (managerUrn == null) { - return corpUser; - } - CorpUser manager = _corpUsers.get(managerUrn); - String managerName = manager.getInfo() != null ? manager.getInfo().getFullName() : null; - if (managerName != null) { - corpUser.getInfo().data().put(MANAGER_FIELD_NAME, managerName); - } - return corpUser; - } - - @Nonnull - public CorpUser getByUserName(@Nonnull String userName) throws Exception { - return get(new CorpuserUrn(userName).toString()); - } - - @Nonnull - public List getAllCorpUsers() throws Exception { - return _corpUsers.getAll(); - } - - public void updateCorpUserEditableConfig(@Nonnull String corpUserUrn, - @Nonnull CorpUserEditableInfo corpUserEditableInfo) throws Exception { - _corpUsers.createEditableInfo(CorpUserUtil.toCorpUserUrn(corpUserUrn), corpUserEditableInfo); - } -} \ No newline at end of file diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DatasetViewDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DatasetViewDao.java deleted file mode 100644 index 7c84e2868c7b7b..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DatasetViewDao.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.linkedin.datahub.dao.view; - -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.common.InstitutionalMemory; -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.datahub.models.view.DatasetColumn; -import com.linkedin.datahub.models.view.DatasetSchema; -import com.linkedin.datahub.models.view.DatasetView; -import com.linkedin.dataset.DatasetDeprecation; -import com.linkedin.schema.SchemaField; -import com.linkedin.schema.SchemaFieldArray; -import com.linkedin.schema.SchemaMetadata; -import com.linkedin.dataset.client.Datasets; -import com.linkedin.dataset.client.Deprecations; -import com.linkedin.dataset.client.Schemas; -import com.linkedin.metadata.snapshot.DatasetSnapshot; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; - -import static com.linkedin.datahub.util.DatasetUtil.toDatasetUrn; -import static com.linkedin.datahub.util.DatasetUtil.toDatasetView; -import static com.linkedin.datahub.util.RestliUtil.toJsonNode; - - -@Slf4j -public class DatasetViewDao { - - private final Datasets _datasets; - private final Deprecations _deprecations; - private final com.linkedin.dataset.client.InstitutionalMemory _institutionalMemory; - private final Schemas _schemas; - - public DatasetViewDao(@Nonnull Datasets datasets, - @Nonnull Deprecations deprecations, - @Nonnull com.linkedin.dataset.client.InstitutionalMemory institutionalMemory, - @Nonnull Schemas schemas) { - this._datasets = datasets; - this._deprecations = deprecations; - this._institutionalMemory = institutionalMemory; - this._schemas = schemas; - } - - @Nonnull - public DatasetView getDatasetView(@Nonnull String datasetUrn) throws Exception { - return toDatasetView(_datasets.get(toDatasetUrn(datasetUrn))); - } - - @Nonnull - public JsonNode getSnapshot(@Nonnull String datasetUrn) throws Exception { - DatasetSnapshot snapshot = _datasets.getLatestFullSnapshot(toDatasetUrn(datasetUrn)); - return toJsonNode(snapshot); - } - - public void updateInstitutionalMemory(@Nonnull String datasetUrn, @Nonnull InstitutionalMemory institutionalMemory) - throws Exception { - _institutionalMemory.updateInstitutionalMemory(toDatasetUrn(datasetUrn), institutionalMemory); - } - - @Nonnull - public JsonNode getInstitutionalMemory(@Nonnull String datasetUrn) throws Exception { - InstitutionalMemory institutionalMemory = _institutionalMemory.getInstitutionalMemory(toDatasetUrn(datasetUrn)); - return toJsonNode(institutionalMemory); - } - - public void setDatasetDeprecation(@Nonnull String datasetUrn, boolean isDeprecated, @Nonnull String deprecationNote, - @Nullable Long decommissionTime, @Nonnull String user) throws Exception { - DatasetDeprecation datasetDeprecation = new DatasetDeprecation() - .setDeprecated(isDeprecated) - .setNote(deprecationNote) - .setActor(new CorpuserUrn(user)); - if (decommissionTime != null) { - datasetDeprecation.setDecommissionTime(decommissionTime); - } - _deprecations.updateDatasetDeprecation(toDatasetUrn(datasetUrn), datasetDeprecation); - } - - /** - * Get dataset schema by dataset urn - * @param datasetUrn String - * @return dataset schema - */ - @Nullable - public DatasetSchema getDatasetSchema(@Nonnull String datasetUrn) throws Exception { - SchemaMetadata schema = _schemas.getLatestSchemaByDataset(toDatasetUrn(datasetUrn)); - if (schema == null) { - return null; - } - SchemaMetadata.PlatformSchema platformSchema = schema.getPlatformSchema(); - - DatasetSchema dsSchema = new DatasetSchema(); - dsSchema.setLastModified(schema.getLastModified().getTime()); - if (platformSchema.isSchemaless()) { - dsSchema.setSchemaless(true); - dsSchema.setColumns(new ArrayList<>()); - return dsSchema; - } - - if (platformSchema.isPrestoDDL()) { - dsSchema.setRawSchema(platformSchema.getPrestoDDL().getRawSchema()); - } else if (platformSchema.isOracleDDL()) { - dsSchema.setRawSchema(platformSchema.getOracleDDL().getTableSchema()); - } else if (platformSchema.isMySqlDDL()) { - dsSchema.setRawSchema(platformSchema.getMySqlDDL().getTableSchema()); - } else if (platformSchema.isKafkaSchema()) { - dsSchema.setRawSchema(platformSchema.getKafkaSchema().getDocumentSchema()); - } else if (platformSchema.isOrcSchema()) { - dsSchema.setRawSchema(platformSchema.getOrcSchema().getSchema()); - } else if (platformSchema.isBinaryJsonSchema()) { - dsSchema.setRawSchema(platformSchema.getBinaryJsonSchema().getSchema()); - } else if (platformSchema.isEspressoSchema()) { - dsSchema.setKeySchema(platformSchema.getEspressoSchema().getTableSchema()); - dsSchema.setRawSchema(platformSchema.getEspressoSchema().getDocumentSchema()); - } else if (platformSchema.isKeyValueSchema()) { - dsSchema.setKeySchema(platformSchema.getKeyValueSchema().getKeySchema()); - dsSchema.setRawSchema(platformSchema.getKeyValueSchema().getValueSchema()); - } - dsSchema.setSchemaless(false); - - if (schema.hasFields() && schema.getFields().size() > 0) { - dsSchema.setColumns(toWhDatasetColumns(schema.getFields())); - } - return dsSchema; - } - - /** - * Convert TMS schema field array to WH list of DatasetColumn - */ - private List toWhDatasetColumns(@Nonnull SchemaFieldArray fields) { - List columns = new ArrayList<>(); - for (SchemaField field : fields) { - DatasetColumn col = new DatasetColumn(); - col.setFieldName(field.getFieldPath()); - col.setFullFieldPath(field.getFieldPath()); - col.setDataType(field.hasNativeDataType() ? field.getNativeDataType() : ""); - col.setComment(field.hasDescription() ? field.getDescription() : ""); - columns.add(col); - } - return columns; - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DocumentSearchDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DocumentSearchDao.java deleted file mode 100644 index e4cbbb0eb20e82..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DocumentSearchDao.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.linkedin.datahub.dao.view; - -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.dataset.client.Datasets; -import com.linkedin.identity.client.CorpUsers; -import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.restli.common.CollectionResponse; -import java.io.IOException; -import java.util.Map; -import javax.annotation.Nonnull; - -import static com.linkedin.datahub.util.RestliUtil.*; - - -public class DocumentSearchDao { - - private final CLIENT _client; - - public DocumentSearchDao(@Nonnull CLIENT client) { - this._client = client; - } - - @Nonnull - public JsonNode search(@Nonnull String input, @Nonnull Map requestFilters, int start, int count) - throws RemoteInvocationException, IOException { - CollectionResponse resp = null; - if (_client instanceof Datasets) { - resp = ((Datasets) _client).search(input, requestFilters, start, count); - } else if (_client instanceof CorpUsers) { - resp = ((CorpUsers) _client).search(input, requestFilters, start, count); - } else { - throw new IllegalStateException("Unexpected client type: " + _client.getClass().getName()); - } - - return collectionResponseWithMetadataToJsonNode(resp); - } - - @Nonnull - public JsonNode autoComplete(@Nonnull String input, @Nonnull String field, - @Nonnull Map requestFilters, int limit) throws RemoteInvocationException, IOException { - AutoCompleteResult autoCompleteResult = null; - if (_client instanceof Datasets) { - autoCompleteResult = ((Datasets) _client).autoComplete(input, field, requestFilters, limit); - } else if (_client instanceof CorpUsers) { - autoCompleteResult = ((CorpUsers) _client).autocomplete(input, field, requestFilters, limit); - } else { - throw new IllegalStateException("Unexpected client type: " + _client.getClass().getName()); - } - - return toJsonNode(autoCompleteResult); - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/OwnerViewDao.java b/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/OwnerViewDao.java deleted file mode 100644 index 3996e190c97388..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/dao/view/OwnerViewDao.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.linkedin.datahub.dao.view; - -import com.linkedin.common.Owner; -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.common.Ownership; -import com.linkedin.common.urn.DatasetUrn; -import com.linkedin.datahub.models.view.DatasetOwner; -import com.linkedin.datahub.models.view.DatasetOwnership; -import com.linkedin.datahub.util.CorpUserUtil; -import com.linkedin.datahub.util.OwnerUtil; -import com.linkedin.dataset.client.Ownerships; -import com.linkedin.identity.CorpUser; -import com.linkedin.identity.client.CorpUsers; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import lombok.extern.slf4j.Slf4j; - -import static com.linkedin.datahub.util.DatasetUtil.*; - -@Slf4j -public class OwnerViewDao { - private final Ownerships _ownerships; - private final CorpUsers _corpUsers; - - public OwnerViewDao(@Nonnull Ownerships ownerships, @Nonnull CorpUsers corpUsers) { - _ownerships = ownerships; - _corpUsers = corpUsers; - } - - @Nonnull - public DatasetOwnership getDatasetOwners(@Nonnull String datasetUrn) throws Exception { - final DatasetUrn urn = toDatasetUrn(datasetUrn); - - Ownership ownership = _ownerships.getLatestOwnership(urn); - Map owners = _corpUsers.batchGet(getOwnerUrns(ownership)); - - DatasetOwnership datasetOwnership = new DatasetOwnership(); - datasetOwnership.setDatasetUrn(datasetUrn); - datasetOwnership.setFromUpstream(false); - datasetOwnership.setOwners(fillDatasetOwner(ownership, owners)); - datasetOwnership.setActor(ownership.getLastModified().getActor().getId()); - datasetOwnership.setLastModified(ownership.getLastModified().getTime()); - return datasetOwnership; - } - - @Nonnull - private List fillDatasetOwner(@Nonnull Ownership ownership, - @Nonnull Map owners) { - final Long modified = ownership.getLastModified().getTime(); - return ownership.getOwners() - .stream() - .map(o -> OwnerUtil.toWhOwner(o, owners.get(getOwnerUrn(o)))) - .peek(o -> o.setModifiedTime(modified)) - .collect(Collectors.toList()); - } - - @Nullable - private CorpuserUrn getOwnerUrn(@Nonnull Owner owner) { - try { - return CorpUserUtil.toCorpUserUrn(owner.getOwner().toString()); - } catch (URISyntaxException e) { - log.error("CorpuserUrn syntax error", e); - return null; - } - } - - @Nonnull - private Set getOwnerUrns(@Nonnull Ownership ownership) { - return ownership.getOwners().stream() - .map(this::getOwnerUrn) - .collect(Collectors.toSet()); - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/PagedCollection.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/PagedCollection.java deleted file mode 100644 index 62dc73ff8f43db..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/PagedCollection.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.linkedin.datahub.models; - -import java.util.List; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - - -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor -public class PagedCollection { - - private int total; - - private int start; - - private int count; - - private List elements; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/CompanyUser.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/table/CompanyUser.java deleted file mode 100644 index 95c2b4fe5a880e..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/CompanyUser.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package com.linkedin.datahub.models.table; - -import lombok.Data; - - -@Data -public class CompanyUser { - - private String userName; - private String pictureLink; - private String displayName; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/Dataset.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/table/Dataset.java deleted file mode 100644 index d7183207ba3169..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/Dataset.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package com.linkedin.datahub.models.table; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.Date; -import java.util.List; - - -public class Dataset { - - public long id; - public String name; - public String source; - public String urn; - public Date created; - public Date modified; - public String formatedModified; - public String schema; - public String hdfsBrowserLink; - public boolean isFavorite; - public boolean isOwned; - public long watchId; - public boolean isWatched; - public boolean hasSchemaHistory; - public JsonNode properties; - public List owners; - public List umpFrequency; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/Group.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/table/Group.java deleted file mode 100644 index 63adce9e573751..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/Group.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package com.linkedin.datahub.models.table; - -import lombok.Data; - - -@Data -public class Group { - - private String name; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/User.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/table/User.java deleted file mode 100644 index 7d0b0dda5d889a..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/User.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package com.linkedin.datahub.models.table; - -import lombok.Data; - - -@Data -public class User { - private int id; - private String userName; - private int departmentNum; - private String email; - private String name; // display name - private String pictureLink; - private UserSetting userSetting; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserEntity.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserEntity.java deleted file mode 100644 index 69d80eec055fad..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserEntity.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package com.linkedin.datahub.models.table; - -import lombok.Data; - -@Data -public class UserEntity { - - public String label; - public String category; - public String displayName; - public String pictureLink; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserSetting.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserSetting.java deleted file mode 100644 index 095088dcc6816c..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserSetting.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package com.linkedin.datahub.models.table; - -import lombok.Data; - - -@Data -public class UserSetting { - - private String detailDefaultView; - private String defaultWatch; -} \ No newline at end of file diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DataOriginView.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DataOriginView.java deleted file mode 100644 index 2598a6abbd2012..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DataOriginView.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.linkedin.datahub.models.view; - -import lombok.Value; - - -/** - * The DataOriginView class is used to find our dataset data origin. Previously, the values EI, CORP, and PROD - * were hardcoded into the frontend UI, but users actually wanted even more information about the cluster the - * data lives on (e.g. Holdem, War). - */ -@Value -public class DataOriginView { - /** - * This will be the item that actually shows up on the UI in the list of possible data origins. It would be the - * more specific designation (e.g. Holdem) - */ - private String displayTitle; - - /** - * Since our urn for datasets isn't to the same degree of specificity as displayTitle, this origin maps to the - * concepts of EI, CORP, PROD so that the UI can understand how to create the link for the dataset fabric - */ - private String origin; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetColumn.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetColumn.java deleted file mode 100644 index 64b7c4352f4254..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetColumn.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.linkedin.datahub.models.view; - -import lombok.Data; -import lombok.NoArgsConstructor; - - -@Data -@NoArgsConstructor -public class DatasetColumn { - - private Long id; - - private int sortID; - - private int parentSortID; - - private String fieldName; - - private String parentPath; - - private String fullFieldPath; - - private String dataType; - - private String comment; - - private Long commentCount; - - private String partitionedStr; - - private boolean partitioned; - - private String nullableStr; - - private boolean nullable; - - private String indexedStr; - - private boolean indexed; - - private String distributedStr; - - private boolean distributed; - - private String treeGridClass; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwner.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwner.java deleted file mode 100644 index 463ac4e0f7a9fe..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwner.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.linkedin.datahub.models.view; - -import java.io.Serializable; -import lombok.Data; - - -@Data -public class DatasetOwner { - - private String userName; - - private String source; - - private String namespace; - - private String name; - - private String email; - - private Boolean isGroup; - - private Boolean isActive; - - private String idType; - - private String type; - - private String subType; - - private Integer sortId; - - private String sourceUrl; - - private String confirmedBy; - - private Long modifiedTime; - - private String pictureLink; - - static class DatasetOwnerKey implements Serializable { - private String userName; - private String source; - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwnership.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwnership.java deleted file mode 100644 index 57b5f4f8b54e3e..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwnership.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.linkedin.datahub.models.view; - -import java.util.List; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - - -@Getter -@Setter -@NoArgsConstructor -public class DatasetOwnership { - - private List owners; - - private Boolean fromUpstream; - - private String datasetUrn; - - private Long lastModified; - - private String actor; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetSchema.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetSchema.java deleted file mode 100644 index dd121abad599c8..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetSchema.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.linkedin.datahub.models.view; - -import java.util.List; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - - -@Setter -@Getter -@NoArgsConstructor -public class DatasetSchema { - - private Boolean schemaless; - - private String rawSchema; - - private String keySchema; - - private List columns; - - private Long lastModified; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetView.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetView.java deleted file mode 100644 index 21625e9ec018cf..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetView.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.linkedin.datahub.models.view; - -import java.util.List; -import java.util.Map; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class DatasetView { - - private String platform; - - private String nativeName; - - private String fabric; - - private String uri; - - private String description; - - private String nativeType; - - private String properties; - - private List tags; - - private Boolean removed; - - private Boolean deprecated; - - private String deprecationNote; - - private Long decommissionTime; - - private Long createdTime; - - private Long modifiedTime; - - private Map customProperties; -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/LineageView.java b/datahub-dao/src/main/java/com/linkedin/datahub/models/view/LineageView.java deleted file mode 100644 index 944a05b9150aaa..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/models/view/LineageView.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.linkedin.datahub.models.view; - -import lombok.Data; - - -@Data -public class LineageView { - - private DatasetView dataset; - - private String type; - - private String actor; - - private String modified; -} \ No newline at end of file diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/util/CorpUserUtil.java b/datahub-dao/src/main/java/com/linkedin/datahub/util/CorpUserUtil.java deleted file mode 100644 index 3db05bdf7cde4f..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/util/CorpUserUtil.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.linkedin.datahub.util; - -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.datahub.models.table.CompanyUser; -import com.linkedin.datahub.models.table.User; -import com.linkedin.datahub.models.table.UserEntity; -import com.linkedin.identity.CorpUser; -import java.net.URISyntaxException; -import javax.annotation.Nonnull; - - -public class CorpUserUtil { - private CorpUserUtil() { - } - - /** - * Deserialize {@link CorpuserUrn} string - * - * @param urnStr urn string - * @return {@link CorpuserUrn} - */ - @Nonnull - public static CorpuserUrn toCorpUserUrn(@Nonnull String urnStr) throws URISyntaxException { - return CorpuserUrn.createFromString(urnStr); - } - - /** - * Convert {@link CorpUser} to {@link User} - * @param corpUser {@link CorpUser} - * @return {@link User} - */ - @Nonnull - public static User toCorpUserView(CorpUser corpUser) { - User user = new User(); - user.setUserName(corpUser.getUsername()); - if (corpUser.getInfo() != null) { - user.setEmail(corpUser.getInfo().getEmail()); - user.setName(corpUser.getInfo().getFullName()); - } - if (corpUser.getEditableInfo() != null) { - user.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString()); - } - return user; - } - - /** - * Convert {@link CorpUser} to {@link CompanyUser} - * @param corpUser {@link CorpUser} - * @return {@link CompanyUser} - */ - @Nonnull - public static CompanyUser toCompanyUserView(CorpUser corpUser) { - CompanyUser user = new CompanyUser(); - user.setUserName(corpUser.getUsername()); - user.setDisplayName(corpUser.getInfo().getDisplayName()); - user.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString()); - return user; - } - - /** - * Convert {@link CorpUser} to {@link UserEntity} - * @param corpUser {@link CorpUser} - * @return {@link UserEntity} - */ - @Nonnull - public static UserEntity toUserEntityView(CorpUser corpUser) { - UserEntity user = new UserEntity(); - user.setCategory("person"); - user.setLabel(corpUser.getUsername()); - if (corpUser.hasInfo()) { - user.setDisplayName(corpUser.getInfo().getDisplayName()); - } - if (corpUser.hasEditableInfo() && corpUser.getEditableInfo().hasPictureLink()) { - user.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString()); - } - return user; - } -} \ No newline at end of file diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/util/DatasetUtil.java b/datahub-dao/src/main/java/com/linkedin/datahub/util/DatasetUtil.java deleted file mode 100644 index 8f912635329116..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/util/DatasetUtil.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.linkedin.datahub.util; - -import com.linkedin.common.AuditStamp; -import com.linkedin.common.urn.DatasetUrn; -import com.linkedin.datahub.models.view.DatasetView; -import com.linkedin.datahub.models.view.LineageView; -import com.linkedin.dataset.Dataset; - -import javax.annotation.Nonnull; -import java.net.URISyntaxException; - -import static com.linkedin.datahub.util.UrnUtil.splitWhUrn; - - -public class DatasetUtil { - private DatasetUtil() { - - } - - /** - * Convert WhereHows dataset URN to DatasetUrn, set dataset origin as PROD. - * @param urn String WH dataset URN - * @return DatasetUrn - */ - public static DatasetUrn toDatasetUrnFromWhUrn(@Nonnull String urn) { - String[] urnParts = splitWhUrn(urn); - return com.linkedin.common.urn.UrnUtils.toDatasetUrn(urnParts[0], urnParts[1], "PROD"); - } - - /** - * Check input string to determine if WH URN or TMS URN, then convert to DatasetUrn - */ - public static DatasetUrn toDatasetUrn(@Nonnull String datasetUrn) throws URISyntaxException { - if (datasetUrn.contains(":///")) { // wherehows URN - return toDatasetUrnFromWhUrn(datasetUrn); - } else { // TMS URN - return DatasetUrn.createFromString(datasetUrn); - } - } - - /** - * Convert TMS Dataset to WH DatasetView - * @param dataset Dataset - * @return DatasetView - */ - public static DatasetView toDatasetView(Dataset dataset) { - DatasetView view = new DatasetView(); - view.setPlatform(dataset.getPlatform().getPlatformNameEntity()); - view.setNativeName(dataset.getName()); - view.setFabric(dataset.getOrigin().name()); - view.setDescription(dataset.getDescription()); - view.setTags(dataset.getTags()); - // construct DatasetUrn and overwrite URI field for frontend use - view.setUri(new DatasetUrn(dataset.getPlatform(), dataset.getName(), dataset.getOrigin()).toString()); - - if (dataset.hasPlatformNativeType()) { - view.setNativeType(dataset.getPlatformNativeType().name()); - } - if (dataset.getStatus() != null) { - view.setRemoved(dataset.getStatus().isRemoved()); - } - if (dataset.hasDeprecation()) { - view.setDeprecated(dataset.getDeprecation().isDeprecated()); - view.setDeprecationNote(dataset.getDeprecation().getNote()); - if (dataset.getDeprecation().hasDecommissionTime()) { - view.setDecommissionTime(dataset.getDeprecation().getDecommissionTime()); - } - } - if (dataset.hasCreated()) { - view.setCreatedTime(dataset.getCreated().getTime()); - } - if (dataset.hasLastModified()) { - view.setModifiedTime(dataset.getLastModified().getTime()); - } - if (dataset.hasProperties()) { - view.setCustomProperties(dataset.getProperties()); - } - return view; - } - - /** - * Converts TMS lineage response to WH LineageView which requires datasetView conversion - * for the dataset in the lineage response - * @param dataset dataset - * @param lineageType type of lineage - * @param auditStamp audit stamp - * @return LineageView - */ - public static LineageView toLineageView(Dataset dataset, String lineageType, AuditStamp auditStamp) { - LineageView view = new LineageView(); - - DatasetView datasetView = toDatasetView(dataset); - datasetView.setModifiedTime(auditStamp.getTime()); - - view.setDataset(datasetView); - view.setType(lineageType); - view.setActor(auditStamp.getActor().toString()); - - return view; - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/util/OwnerUtil.java b/datahub-dao/src/main/java/com/linkedin/datahub/util/OwnerUtil.java deleted file mode 100644 index 20d840d5a83803..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/util/OwnerUtil.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.linkedin.datahub.util; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableBiMap; -import com.linkedin.common.Owner; -import com.linkedin.common.OwnershipType; -import com.linkedin.common.urn.Urn; -import com.linkedin.datahub.models.view.DatasetOwner; -import com.linkedin.identity.CorpUser; -import java.net.URISyntaxException; -import javax.annotation.Nonnull; - - -public class OwnerUtil { - - private OwnerUtil() { - - } - - /** - * Convert from TMS Owner to WhereHows DatasetOwner - * @param owner Owner - * @return DatasetOwner - */ - @VisibleForTesting - public static DatasetOwner toWhOwner(@Nonnull Owner owner, @Nonnull CorpUser corpUser) { - DatasetOwner dsOwner = new DatasetOwner(); - dsOwner.setConfirmedBy("UI"); - dsOwner.setIdType("USER"); - dsOwner.setIsGroup(false); - dsOwner.setNamespace("urn:li:corpuser"); - dsOwner.setSource("UI"); - dsOwner.setUserName(corpUser.getUsername()); - dsOwner.setType(OWNER_CATEGORY_MAP_INV.get(owner.getType())); - - if (corpUser.hasInfo()) { - dsOwner.setEmail(corpUser.getInfo().getEmail()); - dsOwner.setIsActive(corpUser.getInfo().isActive()); - if (corpUser.getInfo().hasFullName()) { - dsOwner.setName(corpUser.getInfo().getFullName()); - } - } - - if (corpUser.hasEditableInfo() && corpUser.getEditableInfo().hasPictureLink()) { - dsOwner.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString()); - } - - return dsOwner; - } - - /** - * Convert from a WhereHows DatasetOwner to TMS Owner - * @param dsOwner dsOwner - * @return Owner - */ - @Nonnull - public static Owner toTmsOwner(@Nonnull DatasetOwner dsOwner) throws URISyntaxException { - return new Owner().setOwner(new Urn(dsOwner.getNamespace() + ":" + dsOwner.getUserName())) - .setType(OWNER_CATEGORY_MAP.get(dsOwner.getType())); - } - - /** - * Mapping between WhereHows owner type values and TMS OwnerCategory - */ - private static final BiMap OWNER_CATEGORY_MAP = - new ImmutableBiMap.Builder() - // format - .put("DataOwner", OwnershipType.DATAOWNER) - .put("Producer", OwnershipType.PRODUCER) - .put("Delegate", OwnershipType.DELEGATE) - .put("Stakeholder", OwnershipType.STAKEHOLDER) - .put("Consumer", OwnershipType.CONSUMER) - .put("Developer", OwnershipType.DEVELOPER) - .build(); - - private static final BiMap OWNER_CATEGORY_MAP_INV = OWNER_CATEGORY_MAP.inverse(); -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/util/RestliUtil.java b/datahub-dao/src/main/java/com/linkedin/datahub/util/RestliUtil.java deleted file mode 100644 index 0bba29e93392c0..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/util/RestliUtil.java +++ /dev/null @@ -1,149 +0,0 @@ -package com.linkedin.datahub.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.linkedin.common.UrnArray; -import com.linkedin.common.urn.Urn; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.datahub.models.PagedCollection; -import com.linkedin.metadata.dao.utils.RecordUtils; -import com.linkedin.metadata.query.SearchResultMetadata; -import com.linkedin.restli.common.CollectionResponse; -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import javax.annotation.Nonnull; - - -public class RestliUtil { - - public static final ObjectMapper OM = new ObjectMapper(); - - private RestliUtil() { - } - - // convert restli collection response to PagedCollection - @Nonnull - public static PagedCollection toPagedCollection( - @Nonnull CollectionResponse response) { - PagedCollection result = new PagedCollection<>(); - result.setElements(response.getElements()); - result.setStart(response.getPaging().getStart()); - result.setCount(response.getPaging().getCount()); - result.setTotal(response.getPaging().getTotal()); - return result; - } - - // convert restli RecordTemplate to JsonNode - @Nonnull - public static JsonNode toJsonNode(@Nonnull T record) throws IOException { - String jsonString = RecordUtils.toJsonString(record); - return OM.readTree(jsonString); - } - - // convert restli collection response to JsonNode - @Nonnull - public static JsonNode collectionResponseToJsonNode( - @Nonnull CollectionResponse response) throws IOException { - final ObjectNode node = OM.createObjectNode(); - node.set("elements", collectionToArrayNode(response.getElements())); - node.put("start", response.getPaging().getStart()); - node.put("count", response.getPaging().getCount()); - node.put("total", response.getPaging().getTotal()); - return node; - } - - // Converts a collection to Json ArrayNode - @Nonnull - public static ArrayNode collectionToArrayNode(@Nonnull Collection elements) - throws IOException { - final ArrayNode arrayNode = OM.createArrayNode(); - for (T element : elements) { - arrayNode.add(toJsonNode(element)); - } - return arrayNode; - } - - // Converts a collection to Json ArrayNode. For each element in the collection, it adds the corresponding urn - @Nonnull - public static ArrayNode collectionToArrayNode(@Nonnull Collection elements, @Nonnull UrnArray urns) - throws IOException { - if (elements.size() != urns.size()) { - throw new RuntimeException("Collection size and urn size should match"); - } - final ArrayNode arrayNode = OM.createArrayNode(); - final Iterator urnIterator = urns.iterator(); - for (T element : elements) { - final ObjectNode node = (ObjectNode) toJsonNode(element); - arrayNode.add(node.put("urn", urnIterator.next().toString())); - } - return arrayNode; - } - - // Convert restli collection response with metadata to JsonNode - @Nonnull - public static JsonNode collectionResponseWithMetadataToJsonNode( - @Nonnull CollectionResponse response) throws IOException { - - final SearchResultMetadata searchResultMetadata = new SearchResultMetadata(response.getMetadataRaw()); - final ObjectNode objectNode = (ObjectNode) collectionResponseToJsonNodeWithoutElements(response); - objectNode.set("elements", collectionToArrayNode(response.getElements(), searchResultMetadata.getUrns())); - - if (response.getMetadataRaw() == null) { - return objectNode; - } - - // Use searchResultMetadatas field as a section in the final result json node - final String name = SearchResultMetadata.fields().searchResultMetadatas().toString(); - final String fieldName = name.startsWith("/") ? name.substring(1) : name; - - objectNode.set(fieldName, toJsonNode(searchResultMetadata).get(fieldName)); - - return objectNode; - } - - // convert restli collection response to JsonNode without populating elements - @Nonnull - public static JsonNode collectionResponseToJsonNodeWithoutElements( - @Nonnull CollectionResponse response) throws IOException { - final ObjectNode node = OM.createObjectNode(); - node.put("start", response.getPaging().getStart()); - node.put("count", response.getPaging().getCount()); - node.put("total", response.getPaging().getTotal()); - return node; - } - - // Convert restli collection response with metadata to JsonNode - @Nonnull - public static JsonNode collectionResponseToJsonNode( - @Nonnull CollectionResponse response, @Nonnull Y metadata) throws IOException { - - JsonNode node = collectionResponseToJsonNode(response); - - ObjectNode objectNode = (ObjectNode) node; - objectNode.set("metadata", toJsonNode(metadata)); - - return objectNode; - } - - // Converts restli map response to JsonNode - @Nonnull - public static JsonNode mapResponseToJsonNode(@Nonnull Map map) throws IOException { - ObjectNode node = OM.createObjectNode(); - for (Map.Entry entry : map.entrySet()) { - node.set(entry.getKey().toString(), toJsonNode(entry.getValue())); - } - return node; - } - - // Converts a string collection to Json ArrayNode - @Nonnull - public static ArrayNode stringCollectionToArrayNode(@Nonnull Collection elements) { - final ArrayNode arrayNode = OM.createArrayNode(); - elements.forEach(s -> arrayNode.add(s)); - return arrayNode; - } -} diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/util/Search.java b/datahub-dao/src/main/java/com/linkedin/datahub/util/Search.java deleted file mode 100644 index 1f665b90abfd11..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/util/Search.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.linkedin.datahub.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.nio.file.Files; -import java.nio.file.Paths; -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -public class Search { - - private static final ObjectMapper _OM = new ObjectMapper(); - - private Search() { - } - - public static String readJsonQueryFile(String jsonFile) { - try { - String contents = new String(Files.readAllBytes(Paths.get(jsonFile))); - JsonNode json = _OM.readTree(contents); - return json.toString(); - } catch (Exception e) { - log.error("ReadJsonQueryFile failed. Error: " + e.getMessage()); - e.printStackTrace(); - return null; - } - } - -} \ No newline at end of file diff --git a/datahub-dao/src/main/java/com/linkedin/datahub/util/UrnUtil.java b/datahub-dao/src/main/java/com/linkedin/datahub/util/UrnUtil.java deleted file mode 100644 index d8e00d75134686..00000000000000 --- a/datahub-dao/src/main/java/com/linkedin/datahub/util/UrnUtil.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.linkedin.datahub.util; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - - -public class UrnUtil { - - private UrnUtil() { - } - - /** - * Get the entity (last part) of an URN, e.g. urn:li:user:abc -> abc - * If input null, return null - * @param urn colon separated URN String - * @return entity String - */ - public static String getUrnEntity(@Nullable String urn) { - return urn == null ? null : urn.substring(urn.lastIndexOf(':') + 1); - } - - /** - * Split WhereHows dataset URN into two parts: platform + dataset name - * Also replace '/' with '.'getObjectArrayListBy in dataset name other than hdfs datasets - * E.g. oracle:///abc/def > [oracle, abc.def] - * @param urn String WhereHows dataset URN - * @return String[] platform + dataset name - */ - public static String[] splitWhUrn(@Nonnull String urn) { - if (urn == null) { - throw new IllegalArgumentException("URN is null"); - } - - final String[] parts = urn.split(":///"); // [platform, dataset name] - - if ("hdfs".equalsIgnoreCase(parts[0])) { - parts[1] = "/" + parts[1]; - } else { - parts[1] = parts[1].replace("/", "."); - } - - return parts; - } - -} diff --git a/datahub-dao/src/main/resources/META-INF/persistence.xml b/datahub-dao/src/main/resources/META-INF/persistence.xml deleted file mode 100644 index df8796cefc29f6..00000000000000 --- a/datahub-dao/src/main/resources/META-INF/persistence.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/datahub-dao/src/test/java/com/linkedin/datahub/dao/view/CorpUserViewDaoTest.java b/datahub-dao/src/test/java/com/linkedin/datahub/dao/view/CorpUserViewDaoTest.java deleted file mode 100644 index 3c929ef37f8705..00000000000000 --- a/datahub-dao/src/test/java/com/linkedin/datahub/dao/view/CorpUserViewDaoTest.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.linkedin.datahub.dao.view; - -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.data.template.StringArray; -import com.linkedin.datahub.util.RestliUtil; -import com.linkedin.identity.CorpUser; -import com.linkedin.identity.CorpUserEditableInfo; -import com.linkedin.identity.CorpUserInfo; -import com.linkedin.identity.client.CorpUsers; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.util.Arrays; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; - -public class CorpUserViewDaoTest { - private CorpUsers _corpUsers; - private CorpUserViewDao _corpUserViewDao; - - @BeforeClass - public void init() { - _corpUsers = mock(CorpUsers.class); - _corpUserViewDao = new CorpUserViewDao(_corpUsers); - } - - @Test - public void testGetManagerName() throws Exception { - String testUser = "testuser"; - String testManager = "testManager"; - CorpuserUrn corpUserUrn = new CorpuserUrn(testUser); - CorpuserUrn corpUserManagerUrn = new CorpuserUrn(testManager); - - CorpUserEditableInfo corpUserEditableInfo = new CorpUserEditableInfo() - .setAboutMe("I am a test user") - .setSkills(new StringArray(Arrays.asList("skill1", "skill2"))) - .setTeams(new StringArray(Arrays.asList("team1", "team2"))); - CorpUser corpManager = new CorpUser().setUsername(testManager) - .setInfo(new CorpUserInfo().setActive(true).setFullName("Test Manager").setEmail("testManager@linkedin.com")); - when(_corpUsers.get(corpUserManagerUrn)).thenReturn(corpManager); - - // test 1: corp user has the manager urn - CorpUserInfo corpUserInfo1 = new CorpUserInfo() - .setActive(true) - .setDisplayName("Test User") - .setEmail("testuser@linkedin.com") - .setManagerUrn(corpUserManagerUrn); - CorpUser corpUser1 = new CorpUser().setUsername(testUser) - .setEditableInfo(corpUserEditableInfo) - .setInfo(corpUserInfo1); - - when(_corpUsers.get(corpUserUrn)).thenReturn(corpUser1); - - JsonNode node1 = RestliUtil.toJsonNode(_corpUserViewDao.get(corpUserUrn.toString())); - assertEquals(node1.get("username").textValue(), testUser); - assertEquals(node1.get("info").get("managerName").textValue(), "Test Manager"); - - // test 2: corp user does not have manager urn - CorpUserInfo corpUserInfo2 = new CorpUserInfo() - .setActive(true) - .setDisplayName("Test User") - .setEmail("testuser@linkedin.com"); - CorpUser corpUser2 = new CorpUser().setUsername(testUser) - .setEditableInfo(corpUserEditableInfo) - .setInfo(corpUserInfo2); - when(_corpUsers.get(corpUserUrn)).thenReturn(corpUser2); - JsonNode node2 = RestliUtil.toJsonNode(_corpUserViewDao.get(corpUserUrn.toString())); - assertEquals(node2.get("username").textValue(), testUser); - assertNull(node2.get("info").get("managerName")); - - // test 3: corp user does not have info set - CorpUser corpUser3 = new CorpUser().setUsername(testUser) - .setEditableInfo(corpUserEditableInfo); - when(_corpUsers.get(corpUserUrn)).thenReturn(corpUser3); - JsonNode node3 = RestliUtil.toJsonNode(_corpUserViewDao.get(corpUserUrn.toString())); - assertEquals(node3.get("username").textValue(), testUser); - assertNull(node3.get("info")); - } -} \ No newline at end of file diff --git a/datahub-dao/src/test/java/com/linkedin/datahub/util/RestliUtilTest.java b/datahub-dao/src/test/java/com/linkedin/datahub/util/RestliUtilTest.java deleted file mode 100644 index 3015cd30a1fa39..00000000000000 --- a/datahub-dao/src/test/java/com/linkedin/datahub/util/RestliUtilTest.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.linkedin.datahub.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.metadata.query.BrowseResultEntity; -import com.linkedin.metadata.query.BrowseResultGroup; -import com.linkedin.metadata.query.BrowseResultMetadata; -import com.linkedin.restli.common.CollectionMetadata; -import com.linkedin.restli.common.CollectionResponse; -import java.util.Collections; -import org.testng.annotations.Test; - -import static com.linkedin.datahub.util.RestliUtil.*; -import static org.testng.Assert.*; - - -public class RestliUtilTest { - - @Test - public void testCollectionResponseToJsonNodeForBrowse() throws Exception { - DataMap documentsDataMap = new DataMap(); - documentsDataMap.put("elements", new DataList(Collections.singletonList(new BrowseResultEntity().data()))); - documentsDataMap.put("paging", new CollectionMetadata().setStart(0).setCount(1).setTotal(10).data()); - CollectionResponse resp = new CollectionResponse<>(documentsDataMap, BrowseResultEntity.class); - - DataMap dataMap = new DataMap(); - dataMap.put("path", "/foo"); - dataMap.put("groups", - new DataList(Collections.singletonList(new BrowseResultGroup().setName("/foo/bar").setCount(31L).data()))); - dataMap.put("totalNumEntities", 100L); - - DataMap metadataMap = new DataMap(); - metadataMap.put("metadata", dataMap); - resp.setMetadataRaw(metadataMap); - - JsonNode node = collectionResponseToJsonNode(resp, new BrowseResultMetadata(dataMap)); - - assertNotNull(node); - assertEquals(node.get("start").asInt(), 0); - assertEquals(node.get("count").asInt(), 1); - assertEquals(node.get("total").asInt(), 10); - assertTrue(node.get("elements").isArray()); - assertEquals(node.get("elements").size(), 1); - assertEquals(node.get("elements").get(0), toJsonNode(new BrowseResultEntity())); - - assertEquals(node.get("metadata").get("path").asText(), "/foo"); - assertTrue(node.get("metadata").get("groups").isArray()); - assertEquals(node.get("metadata").get("groups").size(), 1); - assertEquals(node.get("metadata").get("groups").get(0).get("name").asText(), "/foo/bar"); - assertEquals(node.get("metadata").get("groups").get(0).get("count").asLong(), 31L); - assertEquals(node.get("metadata").get("totalNumEntities").asLong(), 100); - } -} diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java index 0b796bdc889840..1319653668f162 100644 --- a/datahub-frontend/app/controllers/Application.java +++ b/datahub-frontend/app/controllers/Application.java @@ -61,37 +61,6 @@ public Result healthcheck() { return ok("GOOD"); } - @Nonnull - public Result printDeps() { - final String appHome = System.getenv("WHZ_APP_HOME"); - - String libPath = appHome + "/lib"; - String commitFile = appHome + "/commit"; - String commit = ""; - - if (appHome == null) { - return ok("WHZ_APP_HOME environmental variable not defined"); - } - - try (BufferedReader reader = new BufferedReader(new FileReader(commitFile))) { - commit = reader.readLine(); - } catch (IOException ioe) { - _logger.error("Error while reading commit file. Error message: " + ioe.getMessage()); - } - - //get all the files from /libs directory - StringBuilder sb = new StringBuilder(); - - try (Stream paths = Files.list(Paths.get(libPath))) { - paths.filter(Files::isRegularFile). - forEach(f -> sb.append(f.getFileName()).append("\n")); - } catch (IOException ioe) { - _logger.error("Error while traversing the directory. Error message: " + ioe.getMessage()); - } - - return ok("commit: " + commit + "\n" + "libraries: " + sb.toString()); - } - /** * index Action proxies to serveAsset * @@ -171,7 +140,6 @@ private ObjectNode userEntityProps() { * @return Json object with internal wiki links */ @Nonnull - private ObjectNode wikiLinks() { final ObjectNode wikiLinks = Json.newObject(); wikiLinks.put("appHelp", _config.getString("links.wiki.appHelp")); @@ -207,74 +175,4 @@ private ObjectNode trackingInfo() { trackingConfig.put("isEnabled", true); return trackingConfig; } - - @Nonnull - public Result login() { - //You can generate the Csrf token such as String csrfToken = SecurityPlugin.getInstance().getCsrfToken(); - String csrfToken = ""; - return serveAsset(""); - } - - @BodyParser.Of(BodyParser.Json.class) - @Nonnull - public Result authenticate() throws NamingException { - JsonNode json = request().body().asJson(); - // Extract username and password as String from JsonNode, - // null if they are not strings - String username = json.findPath("username").textValue(); - String password = json.findPath("password").textValue(); - - if (StringUtils.isBlank(username)) { - return badRequest("Missing or invalid [username]"); - } - if (password == null) { - password = ""; - } - - session().clear(); - - // Create a uuid string for this session if one doesn't already exist - // to be appended to the Result object - String uuid = session("uuid"); - if (uuid == null) { - uuid = java.util.UUID.randomUUID().toString(); - session("uuid", uuid); - } - - try { - AuthenticationManager.authenticateUser(username, password); - } catch (AuthenticationException e) { - _logger.warn("Authentication error!", e); - return badRequest("Invalid Credential"); - } - - // Adds the username to the session cookie - session("user", username); - - String secretKey = _config.getString("play.http.secret.key"); - try { - //store hashed username to PLAY_SESSION cookie - String hashedUserName = AuthUtil.generateHash(username, secretKey.getBytes()); - session("auth_token", hashedUserName); - } catch (NoSuchAlgorithmException | InvalidKeyException e) { - _logger.error("Failed to hash username", e); - } - - // Construct an ObjectNode with the username and uuid token to be sent with the response - ObjectNode data = Json.newObject(); - data.put("username", username); - data.put("uuid", uuid); - - // Create a new response ObjectNode to return when authenticate request is successful - ObjectNode response = Json.newObject(); - response.put("status", "ok"); - response.set("data", data); - return ok(response); - } - - @Nonnull - public Result logout() { - session().clear(); - return ok(); - } } diff --git a/datahub-frontend/app/controllers/api/v1/Dataset.java b/datahub-frontend/app/controllers/api/v1/Dataset.java deleted file mode 100644 index 5160dff90a7918..00000000000000 --- a/datahub-frontend/app/controllers/api/v1/Dataset.java +++ /dev/null @@ -1,32 +0,0 @@ -package controllers.api.v1; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.linkedin.datahub.dao.DaoFactory; -import com.linkedin.datahub.dao.table.DatasetsDao; -import controllers.Secured; -import play.libs.Json; -import play.mvc.Controller; -import play.mvc.Result; -import play.mvc.Security; - -import javax.annotation.Nonnull; - - -public class Dataset extends Controller { - - private final DatasetsDao _datasetsDao; - - public Dataset() { - _datasetsDao = DaoFactory.getDatasetsDao(); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDatasetOwnerTypes() { - ObjectNode result = Json.newObject(); - - result.put("status", "ok"); - result.set("ownerTypes", Json.toJson(_datasetsDao.getDatasetOwnerTypes())); - return ok(result); - } -} \ No newline at end of file diff --git a/datahub-frontend/app/controllers/api/v1/User.java b/datahub-frontend/app/controllers/api/v1/User.java deleted file mode 100644 index 5cd16b13f56edd..00000000000000 --- a/datahub-frontend/app/controllers/api/v1/User.java +++ /dev/null @@ -1,118 +0,0 @@ -package controllers.api.v1; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.linkedin.datahub.dao.DaoFactory; -import com.linkedin.datahub.dao.view.CorpUserViewDao; -import com.linkedin.datahub.models.table.CompanyUser; -import com.linkedin.datahub.models.table.Group; -import com.linkedin.datahub.models.table.UserEntity; -import com.linkedin.datahub.util.CorpUserUtil; -import controllers.Secured; -import jersey.repackaged.com.google.common.cache.Cache; -import jersey.repackaged.com.google.common.cache.CacheBuilder; -import play.libs.Json; -import play.mvc.Controller; -import play.mvc.Result; -import play.mvc.Security; -import utils.ControllerUtil; - -import javax.annotation.Nonnull; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -public class User extends Controller { - - private static final String CACHE_INTERNAL_USERS = "internal.users.cache"; - private static final String CACHE_INTERNAL_GROUPS = "internal.groups.cache"; - private static final String CACHE_INTERNAL_ENTITIES = "internal.entities.cache"; - - private final CorpUserViewDao _corpUserViewDao; - - private static final Cache CACHE = CacheBuilder.newBuilder() - .expireAfterWrite(24, TimeUnit.HOURS) - .maximumSize(3) - .build(); - - public User() { - _corpUserViewDao = DaoFactory.getCorpUserViewDao(); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getLoggedInUser() { - ObjectNode result = Json.newObject(); - com.linkedin.identity.CorpUser corpUser; - String username = request().attrs().get(Security.USERNAME); - try { - corpUser = _corpUserViewDao.getByUserName(username); - } catch (Exception e) { - throw new RuntimeException(e); - } - - if (corpUser == null || !corpUser.getUsername().equals(username) - || !corpUser.hasInfo()) { - result.put("status", "failed"); - result.put("message", "can't find user info"); - return ok(result); - } - - result.set("user", Json.toJson(CorpUserUtil.toCorpUserView(corpUser))); - result.put("status", "ok"); - return ok(result); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getAllCompanyUsers() { - List users = (List) CACHE.getIfPresent(CACHE_INTERNAL_USERS); - if (users == null || users.size() == 0) { - try { - users = _corpUserViewDao.getAllCorpUsers().stream() - .map(CorpUserUtil::toCompanyUserView) - .collect(Collectors.toList()); - } catch (Exception e) { - return internalServerError(ControllerUtil.errorResponse(e)); - } - CACHE.put(CACHE_INTERNAL_USERS, users); - } - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("employees", Json.toJson(users)); - return ok(result); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getAllGroups() { - List groups = (List) CACHE.getIfPresent(CACHE_INTERNAL_GROUPS); - if (groups == null || groups.size() == 0) { - groups = Collections.emptyList(); - CACHE.put(CACHE_INTERNAL_GROUPS, groups); - } - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("groups", Json.toJson(groups)); - return ok(result); - } - - @Nonnull - public Result getAllUserEntities() { - List entities = (List) CACHE.getIfPresent(CACHE_INTERNAL_ENTITIES); - if (entities == null || entities.size() == 0) { - try { - entities = _corpUserViewDao.getAllCorpUsers().stream() - .map(CorpUserUtil::toUserEntityView) - .collect(Collectors.toList()); - } catch (Exception e) { - return internalServerError(ControllerUtil.errorResponse(e)); - } - CACHE.put(CACHE_INTERNAL_ENTITIES, entities); - } - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("userEntities", Json.toJson(entities)); - return ok(result); - } -} diff --git a/datahub-frontend/app/controllers/api/v2/Browse.java b/datahub-frontend/app/controllers/api/v2/Browse.java deleted file mode 100644 index 8818bb4ec54d14..00000000000000 --- a/datahub-frontend/app/controllers/api/v2/Browse.java +++ /dev/null @@ -1,104 +0,0 @@ -package controllers.api.v2; - -import com.google.common.collect.ImmutableMap; -import com.linkedin.datahub.dao.DaoFactory; -import com.linkedin.datahub.dao.view.BrowseDAO; -import controllers.Secured; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; -import play.Logger; -import play.mvc.Controller; -import play.mvc.Result; -import play.mvc.Security; -import utils.ControllerUtil; - -import javax.annotation.Nonnull; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import static org.apache.commons.lang3.StringUtils.isBlank; - - -public class Browse extends Controller { - - private static final String REQUEST_TYPE = "type"; - private static final String REQUEST_PATH = "path"; - private static final String REQUEST_START = "start"; - private static final String REQUEST_COUNT = "count"; - private static final String REQUEST_URN = "urn"; - - private static final String DATASET_ENTITY_TYPE = "dataset"; - - private static final String DEFAULT_PATH = ""; - private static final int DEFAULT_START_VALUE = 0; - private static final int DEFAULT_PAGE_SIZE = 10; - - private final Map _browseDAOMap; - - private static final ImmutableMap> FILTER_FIELDS = - ImmutableMap.of(); - - private static Map buildRequestMap(String type) { - Map requestMap = new HashMap<>(); - Set facets = FILTER_FIELDS.getOrDefault(type, Collections.emptySet()); - facets.stream().forEach(facet -> requestMap.put(facet, request().getQueryString(facet))); - return requestMap; - } - - public Browse() { - _browseDAOMap = ImmutableMap.builder() - .put(DATASET_ENTITY_TYPE, DaoFactory.getDatasetBrowseDAO()) - .build(); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result browse() { - try { - String type = request().getQueryString(REQUEST_TYPE); - if (isBlank(type)) { - return badRequest("Bad Request. type parameter can not be null"); - } - - String input = StringUtils.defaultIfBlank(request().getQueryString(REQUEST_PATH), DEFAULT_PATH); - int start = NumberUtils.toInt(request().getQueryString(REQUEST_START), DEFAULT_START_VALUE); - int count = NumberUtils.toInt(request().getQueryString(REQUEST_COUNT), DEFAULT_PAGE_SIZE); - - Map requestMap = buildRequestMap(type); - BrowseDAO browseDAO = _browseDAOMap.get(type); - - if (browseDAO != null) { - return ok(browseDAO.browse(input, requestMap, start, count)); - } else { - return badRequest("Bad Request. Type is not supported"); - } - } catch (Exception e) { - Logger.error("Browse failed: ", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getBrowsePaths() { - try { - String type = request().getQueryString(REQUEST_TYPE); - String urn = request().getQueryString(REQUEST_URN); - if (isBlank(type) || isBlank(urn)) { - return badRequest("Bad Request. Type or urn parameter can not be null"); - } - - BrowseDAO browseDAO = _browseDAOMap.get(type); - if (browseDAO != null) { - return ok(browseDAO.getBrowsePaths(urn)); - } else { - return badRequest("Bad Request. Type is not supported"); - } - } catch (Exception e) { - Logger.error("Browse paths request failed: ", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } -} diff --git a/datahub-frontend/app/controllers/api/v2/CorpUser.java b/datahub-frontend/app/controllers/api/v2/CorpUser.java deleted file mode 100644 index bf77a843728295..00000000000000 --- a/datahub-frontend/app/controllers/api/v2/CorpUser.java +++ /dev/null @@ -1,87 +0,0 @@ -package controllers.api.v2; - -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.datahub.dao.DaoFactory; -import com.linkedin.datahub.dao.view.CorpUserViewDao; -import com.linkedin.datahub.util.CorpUserUtil; -import com.linkedin.identity.CorpUserEditableInfo; -import com.linkedin.metadata.dao.utils.RecordUtils; -import controllers.Secured; -import org.apache.commons.lang3.StringUtils; -import play.Logger; -import play.libs.Json; -import play.mvc.Controller; -import play.mvc.Result; -import play.mvc.Results; -import play.mvc.Security; -import utils.ControllerUtil; - -import javax.annotation.Nonnull; -import java.net.URISyntaxException; - -import static com.linkedin.datahub.util.RestliUtil.*; - -public class CorpUser extends Controller { - - private static final JsonNode EMPTY_RESPONSE = Json.newObject(); - - private final CorpUserViewDao _corpUserViewDao; - - public CorpUser() { - _corpUserViewDao = DaoFactory.getCorpUserViewDao(); - } - - /** - * Get CorpUser given corpUser urn - * @param corpUserUrn String - * @return CorpUser - */ - @Security.Authenticated(Secured.class) - @Nonnull - public Result getCorpUser(@Nonnull String corpUserUrn) { - try { - return Results.ok(toJsonNode(_corpUserViewDao.get(corpUserUrn))); - } catch (Exception e) { - if (e.toString().contains("Response status 404")) { - return notFound(EMPTY_RESPONSE); - } - - Logger.error("Failed to get corp user", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } - - /** - * Creates or Updates CorpUserEditableInfo aspect - * - * @param corpUserUrn CorpUser urn - */ - @Security.Authenticated(Secured.class) - @Nonnull - public Result updateCorpUserEditableInfo(@Nonnull String corpUserUrn) { - final String username = session("user"); - if (StringUtils.isBlank(username)) { - return unauthorized(EMPTY_RESPONSE); - } - final CorpuserUrn corpUser; - try { - corpUser = CorpUserUtil.toCorpUserUrn(corpUserUrn); - } catch (URISyntaxException e) { - return unauthorized("Invalid urn"); - } - if (!corpUser.getUsernameEntity().equals(username)) { - return unauthorized(EMPTY_RESPONSE); - } - final JsonNode requestBody = request().body().asJson(); - try { - CorpUserEditableInfo corpUserEditableInfo = - RecordUtils.toRecordTemplate(CorpUserEditableInfo.class, requestBody.toString()); - _corpUserViewDao.updateCorpUserEditableConfig(corpUserUrn, corpUserEditableInfo); - return Results.ok((JsonNode) Json.newObject().set("updated", Json.toJson(true))); - } catch (Exception e) { - Logger.error("Failed to upsert corp user editable info", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } -} \ No newline at end of file diff --git a/datahub-frontend/app/controllers/api/v2/Dataset.java b/datahub-frontend/app/controllers/api/v2/Dataset.java deleted file mode 100644 index 5198ffc5dfb36b..00000000000000 --- a/datahub-frontend/app/controllers/api/v2/Dataset.java +++ /dev/null @@ -1,286 +0,0 @@ -package controllers.api.v2; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.common.AuditStamp; -import com.linkedin.common.InstitutionalMemory; -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.datahub.dao.DaoFactory; -import com.linkedin.datahub.dao.table.DataPlatformsDao; -import com.linkedin.datahub.dao.table.DatasetOwnerDao; -import com.linkedin.datahub.dao.table.LineageDao; -import com.linkedin.datahub.dao.view.DatasetViewDao; -import com.linkedin.datahub.dao.view.OwnerViewDao; -import com.linkedin.datahub.models.view.*; -import com.linkedin.metadata.dao.utils.RecordUtils; -import controllers.Secured; -import org.apache.commons.lang3.StringUtils; -import play.Logger; -import play.libs.Json; -import play.mvc.Controller; -import play.mvc.Result; -import play.mvc.Security; -import utils.ControllerUtil; - -import javax.annotation.Nonnull; -import java.util.List; - -public class Dataset extends Controller { - - private final DatasetViewDao _datasetViewDao; - private final OwnerViewDao _ownerViewDao; - private final DatasetOwnerDao _datasetOwnerDao; - private final LineageDao _lineageDao; - private final DataPlatformsDao _dataPlatformsDao; - - private static final JsonNode EMPTY_RESPONSE = Json.newObject(); - - public Dataset() { - _datasetViewDao = DaoFactory.getDatasetViewDao(); - _ownerViewDao = DaoFactory.getOwnerViewDao(); - _datasetOwnerDao = DaoFactory.getDatasetOwnerDao(); - _lineageDao = DaoFactory.getLineageDao(); - _dataPlatformsDao = DaoFactory.getDataPlatformsDao(); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDataset(@Nonnull String datasetUrn) { - final DatasetView view; - try { - view = _datasetViewDao.getDatasetView(datasetUrn); - } catch (Exception e) { - if (ControllerUtil.checkErrorCode(e, 404)) { - return notFound(EMPTY_RESPONSE); - } - - Logger.error("Failed to get dataset view", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - - return ok(Json.toJson(view)); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result updateDatasetDeprecation(@Nonnull String datasetUrn) { - final String username = session("user"); - if (StringUtils.isBlank(username)) { - return unauthorized(EMPTY_RESPONSE); - } - - try { - JsonNode record = request().body().asJson(); - - boolean deprecated = record.get("deprecated").asBoolean(); - String deprecationNote = record.hasNonNull("deprecationNote") ? record.get("deprecationNote").asText() : ""; - long decommissionTime = record.hasNonNull("decommissionTime") ? record.get("decommissionTime").asLong() : 0; - if (deprecated && decommissionTime <= 0) { - throw new IllegalArgumentException("Invalid decommission time"); - } - - _datasetViewDao.setDatasetDeprecation(datasetUrn, deprecated, deprecationNote, decommissionTime, username); - } catch (Exception e) { - Logger.error("Update dataset deprecation fail", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - - return ok(EMPTY_RESPONSE); - } - - /** - * Creates or Updates {@link InstitutionalMemory} aspect given dataset urn - * - *

{@link com.linkedin.common.InstitutionalMemoryMetadata} record that does not contain audit stamp is filled - * here with session username as the actor and system time as the time

- * - * @param datasetUrn Dataset Urn - */ - @Security.Authenticated(Secured.class) - @Nonnull - public Result updateInstitutionalMemory(@Nonnull String datasetUrn) { - final String username = session("user"); - if (StringUtils.isBlank(username)) { - return unauthorized(EMPTY_RESPONSE); - } - final JsonNode requestBody = request().body().asJson(); - try { - InstitutionalMemory institutionalMemory = - RecordUtils.toRecordTemplate(InstitutionalMemory.class, requestBody.toString()); - institutionalMemory.getElements().forEach(element -> { - if (!element.hasCreateStamp()) { - element.setCreateStamp( - new AuditStamp().setActor(new CorpuserUrn(username)).setTime(System.currentTimeMillis())); - } - }); - _datasetViewDao.updateInstitutionalMemory(datasetUrn, institutionalMemory); - } catch (Exception e) { - Logger.error("Failed to update Institutional Memory aspect", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - return ok(EMPTY_RESPONSE); - } - - /** - * Gets {@link InstitutionalMemory} aspect given dataset urn - * - * @param datasetUrn Dataset Urn - */ - @Security.Authenticated(Secured.class) - @Nonnull - public Result getInstitutionalMemory(String datasetUrn) { - try { - JsonNode responsenode = _datasetViewDao.getInstitutionalMemory(datasetUrn); - return ok(responsenode); - } catch (Exception e) { - if (ControllerUtil.checkErrorCode(e, 404)) { - return notFound(EMPTY_RESPONSE); - } - Logger.error("Failed to get Institutional Memory aspect", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDatasetOwners(@Nonnull String datasetUrn) { - final DatasetOwnership ownership; - try { - ownership = _ownerViewDao.getDatasetOwners(datasetUrn); - } catch (Exception e) { - if (ControllerUtil.checkErrorCode(e, 404)) { - return notFound(EMPTY_RESPONSE); - } - - Logger.error("Fetch owners fail", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - return ok(Json.toJson(ownership)); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result updateDatasetOwners(@Nonnull String datasetUrn) { - final String username = session("user"); - if (StringUtils.isBlank(username)) { - return unauthorized(EMPTY_RESPONSE); - } - - final JsonNode content = request().body().asJson(); - // content should contain arraynode 'owners': [] - if (content == null || !content.has("owners") || !content.get("owners").isArray()) { - return badRequest(ControllerUtil.errorResponse("Update dataset owners fail: missing owners field")); - } - - try { - final List owners = Json.mapper().readerFor(new TypeReference>() { - }).readValue(content.get("owners")); - - long confirmedOwnerUserCount = owners.stream() - .filter(s -> "DataOwner".equalsIgnoreCase(s.getType()) && "user".equalsIgnoreCase(s.getIdType()) - && "UI".equalsIgnoreCase(s.getSource())) - .count(); - - // enforce at least two UI (confirmed) USER DataOwner for a dataset before making any changes - if (confirmedOwnerUserCount < 2) { - return badRequest(ControllerUtil.errorResponse("Less than 2 UI USER owners")); - } - - _datasetOwnerDao.updateDatasetOwners(datasetUrn, owners, username); - } catch (Exception e) { - Logger.error("Update Dataset owners fail", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - return ok(EMPTY_RESPONSE); - } - - /** - * Gets latest DatasetSnapshot given DatasetUrn - * - * @param datasetUrn String - * @return DatasetSnapshot - */ - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDatasetSnapshot(@Nonnull String datasetUrn) { - try { - return ok(_datasetViewDao.getSnapshot(datasetUrn)); - } catch (Exception e) { - if (e.toString().contains("Response status 404")) { - return notFound(EMPTY_RESPONSE); - } - - Logger.error("Failed to get dataset snapshot", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDatasetSchema(@Nonnull String datasetUrn) { - final DatasetSchema schema; - try { - schema = _datasetViewDao.getDatasetSchema(datasetUrn); - } catch (Exception e) { - if (ControllerUtil.checkErrorCode(e, 404)) { - return notFound(EMPTY_RESPONSE); - } - - Logger.error("Fetch schema fail", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - - if (schema == null) { - return notFound(EMPTY_RESPONSE); - } - return ok((JsonNode) Json.newObject().set("schema", Json.toJson(schema))); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDatasetSuggestedOwners(@Nonnull String datasetUrn) { - return ok(EMPTY_RESPONSE); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDatasetUpstreams(@Nonnull String datasetUrn) { - final List upstreams; - try { - upstreams = _lineageDao.getUpstreamLineage(datasetUrn); - } catch (Exception e) { - if (ControllerUtil.checkErrorCode(e, 404)) { - int[] emptyUpstreams = new int[0]; - return ok(Json.toJson(emptyUpstreams)); - } - - Logger.error("Fetch Dataset upstreams error", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - return ok(Json.toJson(upstreams)); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDatasetDownstreams(@Nonnull String datasetUrn) { - final List downstreams; - try { - downstreams = _lineageDao.getDownstreamLineage(datasetUrn); - } catch (Exception e) { - Logger.error("Fetch Dataset downstreams error", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - return ok(Json.toJson(downstreams)); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result getDataPlatforms() { - try { - return ok(ControllerUtil.jsonNode("platforms", _dataPlatformsDao.getAllPlatforms())); - } catch (final Exception e) { - Logger.error("Fail to get data platforms", e); - return notFound(ControllerUtil.errorResponse(e)); - } - } -} diff --git a/datahub-frontend/app/controllers/api/v2/Search.java b/datahub-frontend/app/controllers/api/v2/Search.java deleted file mode 100644 index 853f55a9f8fac8..00000000000000 --- a/datahub-frontend/app/controllers/api/v2/Search.java +++ /dev/null @@ -1,135 +0,0 @@ -package controllers.api.v2; - -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.linkedin.datahub.dao.DaoFactory; -import com.linkedin.datahub.dao.view.DocumentSearchDao; -import controllers.Secured; -import org.apache.commons.lang3.math.NumberUtils; -import play.Logger; -import play.mvc.Controller; -import play.mvc.Result; -import play.mvc.Security; -import utils.ControllerUtil; -import utils.SearchUtil; - -import javax.annotation.Nonnull; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import static org.apache.commons.lang3.StringUtils.isBlank; - -public class Search extends Controller { - private static final String REQUEST_TYPE = "type"; - private static final String REQUEST_INPUT = "input"; - private static final String REQUEST_FIELD = "field"; - private static final String REQUEST_START = "start"; - private static final String REQUEST_COUNT = "count"; - private static final String REQUEST_LIMIT = "limit"; - - private static final String CORP_USER_TYPE = "corpuser"; - private static final String DATASET_TYPE = "dataset"; - - private static final Set CORP_USER_FACET_FIELDS = Collections.emptySet(); - private static final Set DATASET_FACET_FIELDS = - ImmutableSet.of("origin", "platform"); - private static final ImmutableMap> FACET_FIELDS = ImmutableMap.of( - CORP_USER_TYPE, CORP_USER_FACET_FIELDS, - DATASET_TYPE, DATASET_FACET_FIELDS - ); - - private static final int _DEFAULT_START_VALUE = 0; - private static final int _DEFAULT_PAGE_SIZE = 10; - private static final int _DEFAULT_LIMIT_VALUE = 20; - - private static Map buildRequestMap(String type) { - Map requestMap = new HashMap<>(); - Set facets = FACET_FIELDS.getOrDefault(type, Collections.emptySet()); - facets.stream().forEach(facet -> requestMap.put(facet, request().getQueryString(facet))); - return requestMap; - } - - private final Map _searchDaoMap; - - public Search() { - _searchDaoMap = ImmutableMap.builder() - .put(CORP_USER_TYPE, DaoFactory.getCorpUserDocumentSearchDao()) - .put(DATASET_TYPE, DaoFactory.getDatasetDocumentSearchDao()) - .build(); - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result search() { - try { - String type = request().getQueryString(REQUEST_TYPE); - if (isBlank(type)) { - return badRequest("Bad Request. type parameter can not be null or empty"); - } - - // escape forward slash since it is a reserved character in Elasticsearch - // TODO: Once apis for exact or advanced(with support for field specific/regex) search are exposed, - // update to call appropriate api based on indication from user. - final String input = SearchUtil - .escapeForwardSlash(Strings.nullToEmpty(request().getQueryString(REQUEST_INPUT))); - if (isBlank(input)) { - return badRequest("Bad Request. input parameter can not be null or empty"); - } - - int start = NumberUtils.toInt(request().getQueryString(REQUEST_START), _DEFAULT_START_VALUE); - int count = NumberUtils.toInt(request().getQueryString(REQUEST_COUNT), _DEFAULT_PAGE_SIZE); - - Map requestMap = buildRequestMap(type); - DocumentSearchDao documentSearchDao = _searchDaoMap.get(type); - - if (documentSearchDao != null) { - return ok(documentSearchDao.search(input, requestMap, start, count)); - } else { - return badRequest("Bad Request. Type is not supported"); - } - } catch (Exception e) { - Logger.error("Failed in documents search.", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } - - @Security.Authenticated(Secured.class) - @Nonnull - public Result autoComplete() { - try { - - String type = request().getQueryString(REQUEST_TYPE); - if (isBlank(type)) { - return badRequest("Bad Request. type parameter can not be null or empty"); - } - - String field = request().getQueryString(REQUEST_FIELD); - if (isBlank(field)) { - return badRequest("Bad Request. field parameter can not be null or empty"); - } - - // escape forward slash since it is a reserved character in Elasticsearch - // TODO: Once apis for exact or advanced(with support for field specific/regex) search are exposed, - // update to call appropriate api based on indication from user. - final String input = SearchUtil - .escapeForwardSlash(Strings.nullToEmpty(request().getQueryString(REQUEST_INPUT))); - - int limit = NumberUtils.toInt(request().getQueryString(REQUEST_LIMIT), _DEFAULT_LIMIT_VALUE); - - Map requestMap = buildRequestMap(type); - DocumentSearchDao documentSearchDao = _searchDaoMap.get(type); - - if (documentSearchDao != null) { - return ok(documentSearchDao.autoComplete(input, field, requestMap, limit)); - } else { - return badRequest("Bad Request. Type is not supported"); - } - } catch (Exception e) { - Logger.error("Failed in auto complete request.", e); - return internalServerError(ControllerUtil.errorResponse(e)); - } - } -} diff --git a/datahub-frontend/app/utils/ControllerUtil.java b/datahub-frontend/app/utils/ControllerUtil.java deleted file mode 100644 index 11a31deb4223e7..00000000000000 --- a/datahub-frontend/app/utils/ControllerUtil.java +++ /dev/null @@ -1,58 +0,0 @@ -package utils; - -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.datahub.util.RestliUtil; -import com.linkedin.restli.client.RestLiResponseException; -import java.io.IOException; -import javax.annotation.Nonnull; -import play.Logger; -import play.libs.Json; - - -/** - * Helper class for controller APIs - */ -public class ControllerUtil { - - private ControllerUtil() { - //utility class - } - - @Nonnull - public static JsonNode errorResponse(@Nonnull E e) { - return errorResponse(e.toString()); - } - - @Nonnull - public static JsonNode errorResponse(@Nonnull String msg) { - return Json.newObject().put("msg", msg); - } - - public static boolean checkErrorCode(Exception e, int statusCode) { - return (e instanceof RestLiResponseException) && (((RestLiResponseException) e).getStatus() == statusCode); - } - - /** - * Creates a generic jsonNode out of a key and value which is a very common use case - * @param key - * @param value - * @return - */ - @Nonnull - public static JsonNode jsonNode(@Nonnull final String key, @Nonnull final Object value) { - JsonNode node = null; - if (value instanceof RecordTemplate) { - try { - node = RestliUtil.toJsonNode((RecordTemplate) value); - } catch (final IOException e) { - Logger.error("Could not create a json", e); - } - } else if (value instanceof JsonNode) { - node = (JsonNode) value; - } else { - node = Json.toJson(value); - } - return Json.newObject().set(key, node); - } -} \ No newline at end of file diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes index 39b76d580ca0a6..7ff5871fb026e1 100644 --- a/datahub-frontend/conf/routes +++ b/datahub-frontend/conf/routes @@ -7,36 +7,7 @@ GET / controllers.Application.index(path="index.html") GET /admin controllers.Application.healthcheck() -POST /authenticate controllers.Application.authenticate() GET /config controllers.Application.appConfig() -GET /deps controllers.Application.printDeps() -GET /login controllers.Application.login() -GET /logout controllers.Application.logout() - -GET /api/v1/owner/types controllers.api.v1.Dataset.getDatasetOwnerTypes() -GET /api/v1/party/employees controllers.api.v1.User.getAllCompanyUsers() -GET /api/v1/party/entities controllers.api.v1.User.getAllUserEntities() -GET /api/v1/party/groups controllers.api.v1.User.getAllGroups() -GET /api/v1/user/me controllers.api.v1.User.getLoggedInUser() - -GET /api/v2/autocomplete controllers.api.v2.Search.autoComplete() -GET /api/v2/browse controllers.api.v2.Browse.browse() -GET /api/v2/browsePaths controllers.api.v2.Browse.getBrowsePaths() -GET /api/v2/corpusers/:urn controllers.api.v2.CorpUser.getCorpUser(urn: String) -POST /api/v2/corpusers/:urn/editableInfo controllers.api.v2.CorpUser.updateCorpUserEditableInfo(urn: String) -GET /api/v2/datasets/:urn controllers.api.v2.Dataset.getDataset(urn: String) -PUT /api/v2/datasets/:urn/deprecate controllers.api.v2.Dataset.updateDatasetDeprecation(urn: String) -GET /api/v2/datasets/:urn/downstreams controllers.api.v2.Dataset.getDatasetDownstreams(urn: String) -GET /api/v2/datasets/:urn/institutionalmemory controllers.api.v2.Dataset.getInstitutionalMemory(urn: String) -POST /api/v2/datasets/:urn/institutionalmemory controllers.api.v2.Dataset.updateInstitutionalMemory(urn: String) -GET /api/v2/datasets/:urn/owners controllers.api.v2.Dataset.getDatasetOwners(urn: String) -POST /api/v2/datasets/:urn/owners controllers.api.v2.Dataset.updateDatasetOwners(urn: String) -GET /api/v2/datasets/:urn/owners/suggestion controllers.api.v2.Dataset.getDatasetSuggestedOwners(urn: String) -GET /api/v2/datasets/:urn/schema controllers.api.v2.Dataset.getDatasetSchema(urn: String) -GET /api/v2/datasets/:urn/snapshot controllers.api.v2.Dataset.getDatasetSnapshot(urn: String) -GET /api/v2/datasets/:urn/upstreams controllers.api.v2.Dataset.getDatasetUpstreams(urn: String) -GET /api/v2/list/platforms controllers.api.v2.Dataset.getDataPlatforms -GET /api/v2/search controllers.api.v2.Search.search() # Routes used exclusively by the React application. diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index 086d2434843e31..5bd0a19d1b65b6 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -27,7 +27,6 @@ dependencies { play('net.minidev:json-smart:2.4.1') play('io.netty:netty-all:4.1.44.Final') } - play project(":datahub-dao") play project(":datahub-graphql-core") play externalDependency.jettyJaas From 5e3904f45ca9e85ada6e3e614b0de590052c2ada Mon Sep 17 00:00:00 2001 From: John Joyce Date: Mon, 2 Aug 2021 22:09:08 -0700 Subject: [PATCH 2/7] Additional removals --- docker/docker-compose.ember.yml | 19 ------------------- settings.gradle | 1 - 2 files changed, 20 deletions(-) delete mode 100644 docker/docker-compose.ember.yml diff --git a/docker/docker-compose.ember.yml b/docker/docker-compose.ember.yml deleted file mode 100644 index 5779508f05f91e..00000000000000 --- a/docker/docker-compose.ember.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Introduces datahub-frontend-ember service to serve the Ember UI on localhost:9001/ ---- -version: '3.8' -services: - datahub-frontend-ember: - build: - context: ../ - dockerfile: docker/datahub-frontend/Dockerfile - args: - SERVER_PORT: 9001 - ENABLE_EMBER: "true" - image: linkedin/datahub-frontend:${DATAHUB_VERSION:-head} - env_file: datahub-frontend/env/docker.env - hostname: datahub-frontend - container_name: datahub-frontend - ports: - - "9001:9001" - depends_on: - - datahub-gms diff --git a/settings.gradle b/settings.gradle index 1d0a13f1104a38..248cd0b2f1bc24 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,3 @@ -include 'datahub-dao' include 'datahub-frontend' include 'datahub-graphql-core' include 'datahub-gms-graphql-service' From a983380f55dd3cd17d5d468cbd78fca9c13cc873 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 3 Aug 2021 15:45:27 -0700 Subject: [PATCH 3/7] Updating Smoke Tests to reflect latest state of frontend and backend services --- gms/client/build.gradle | 1 - metadata-jobs/mae-consumer/build.gradle | 1 - metadata-jobs/mce-consumer/build.gradle | 1 - smoke-test/test_e2e.py | 260 +++++++++++++++--------- 4 files changed, 161 insertions(+), 102 deletions(-) diff --git a/gms/client/build.gradle b/gms/client/build.gradle index 4323a4f28e8899..6dab9f9f94879c 100644 --- a/gms/client/build.gradle +++ b/gms/client/build.gradle @@ -4,7 +4,6 @@ dependencies { compile project(':gms:api') compile project(':gms:impl') compile project(path: ':gms:api', configuration: 'restClient') - compile project(':metadata-dao-impl:restli-dao') compile project(':metadata-events:mxe-schemas') dataModel project(':li-utils') diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle index 48d8413cf81704..c8130dadb3d6c2 100644 --- a/metadata-jobs/mae-consumer/build.gradle +++ b/metadata-jobs/mae-consumer/build.gradle @@ -19,7 +19,6 @@ dependencies { compile project(':metadata-utils') compile project(":entity-registry") compile project(':metadata-builders') - compile project(':metadata-dao-impl:restli-dao') compile project(':metadata-events:mxe-schemas') compile project(':metadata-events:mxe-avro-1.7') compile project(':metadata-events:mxe-registration') diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle index a895437841f531..d6d60a27b4a0d4 100644 --- a/metadata-jobs/mce-consumer/build.gradle +++ b/metadata-jobs/mce-consumer/build.gradle @@ -18,7 +18,6 @@ dependencies { compile project(':metadata-events:mxe-avro-1.7') compile project(':metadata-events:mxe-registration') compile project(':metadata-events:mxe-utils-avro-1.7') - compile project(':metadata-dao-impl:restli-dao') compile project(':metadata-io') compile project(':gms:client') compile spec.product.pegasus.restliClient diff --git a/smoke-test/test_e2e.py b/smoke-test/test_e2e.py index 0cd74b8d3a7887..f48ed401fe416c 100644 --- a/smoke-test/test_e2e.py +++ b/smoke-test/test_e2e.py @@ -2,6 +2,7 @@ import pytest import requests +import urllib from datahub.cli.docker import check_local_docker_containers from datahub.ingestion.run.pipeline import Pipeline @@ -98,41 +99,12 @@ def test_run_ingestion(wait_for_healthchecks): pass -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_gms_list_data_platforms(): - response = requests.get( - f"{GMS_ENDPOINT}/dataPlatforms", - headers={ - **restli_default_headers, - "X-RestLi-Method": "get_all", - }, - ) - response.raise_for_status() - data = response.json() - - assert len(data["elements"]) > 10 - - -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_gms_get_all_users(): - response = requests.get( - f"{GMS_ENDPOINT}/corpUsers", - headers={ - **restli_default_headers, - "X-RestLi-Method": "get_all", - }, - ) - response.raise_for_status() - data = response.json() - - assert len(data["elements"]) >= 3 - - @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_gms_get_user(): username = "jdoe" + urn = f"urn:li:corpuser:{username}" response = requests.get( - f"{GMS_ENDPOINT}/corpUsers/($params:(),name:{username})", + f"{GMS_ENDPOINT}/entities/{urllib.parse.quote(urn)}", headers={ **restli_default_headers, }, @@ -140,9 +112,9 @@ def test_gms_get_user(): response.raise_for_status() data = response.json() - assert data["username"] == username - assert data["info"]["displayName"] - assert data["info"]["email"] + assert data["value"] + assert data["value"]["com.linkedin.metadata.snapshot.CorpUserSnapshot"] + assert data["value"]["com.linkedin.metadata.snapshot.CorpUserSnapshot"]["urn"] == urn @pytest.mark.parametrize( @@ -172,19 +144,18 @@ def test_gms_get_dataset(platform, dataset_name, env): urn = f"urn:li:dataset:({platform},{dataset_name},{env})" response = requests.get( - f"{GMS_ENDPOINT}/datasets/($params:(),name:{dataset_name},origin:{env},platform:{requests.utils.quote(platform)})", + f"{GMS_ENDPOINT}/entities/{urllib.parse.quote(urn)}", headers={ **restli_default_headers, "X-RestLi-Method": "get", }, ) response.raise_for_status() - data = response.json() + res_data = response.json() - assert data["urn"] == urn - assert data["name"] == dataset_name - assert data["platform"] == platform - assert len(data["schemaMetadata"]["fields"]) >= 2 + assert res_data["value"] + assert res_data["value"]["com.linkedin.metadata.snapshot.DatasetSnapshot"] + assert res_data["value"]["com.linkedin.metadata.snapshot.DatasetSnapshot"]["urn"] == urn @pytest.mark.parametrize( @@ -196,19 +167,26 @@ def test_gms_get_dataset(platform, dataset_name, env): ) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_gms_search_dataset(query, min_expected_results): - response = requests.get( - f"{GMS_ENDPOINT}/datasets?q=search&input={query}", - headers={ - **restli_default_headers, - "X-RestLi-Method": "finder", - }, + + + json = { + "input": f"{query}", + "entity": "dataset", + "start": 0, + "count": 10 + } + print(json) + response = requests.post( + f"{GMS_ENDPOINT}/entities?action=search", + headers=restli_default_headers, + json=json ) response.raise_for_status() - data = response.json() + res_data = response.json() - assert len(data["elements"]) >= min_expected_results - assert data["paging"]["total"] >= min_expected_results - assert data["elements"][0]["urn"] + assert res_data["value"] + assert res_data["value"]["numEntities"] >= min_expected_results + assert len(res_data["value"]["entities"]) >= min_expected_results @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @@ -253,7 +231,7 @@ def frontend_session(wait_for_healthchecks): } data = '{"username":"datahub", "password":"datahub"}' response = session.post( - f"{FRONTEND_ENDPOINT}/authenticate", headers=headers, data=data + f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data ) response.raise_for_status() @@ -267,15 +245,43 @@ def test_frontend_auth(frontend_session): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_frontend_browse_datasets(frontend_session): - response = frontend_session.get( - f"{FRONTEND_ENDPOINT}/api/v2/browse?type=dataset&path=/prod" + + json = { + "query": """query browse($input: BrowseInput!) {\n + browse(input: $input) {\n + start\n + count\n + total\n + groups { + name + } + entities {\n + ... on Dataset {\n + urn\n + name\n + }\n + }\n + }\n + }""", + "variables": { + "input": { + "type": "DATASET", + "path": ["prod"] + } + } + } + + response = frontend_session.post( + f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json ) - response.raise_for_status() - data = response.json() - assert data["metadata"]["totalNumEntities"] >= 4 - assert len(data["metadata"]["groups"]) >= 4 - assert len(data["metadata"]["groups"]) <= 8 + response.raise_for_status() + res_data = response.json() + assert res_data + assert res_data["data"] + assert res_data["data"]["browse"] + assert len(res_data["data"]["browse"]["entities"]) == 0 + assert len(res_data["data"]["browse"]["groups"]) > 0 @pytest.mark.parametrize( @@ -286,36 +292,81 @@ def test_frontend_browse_datasets(frontend_session): ], ) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_frontend_browse_datasets(frontend_session, query, min_expected_results): - response = frontend_session.get( - f"{FRONTEND_ENDPOINT}/api/v2/search?type=dataset&input={query}" - ) - response.raise_for_status() - data = response.json() - - assert len(data["elements"]) >= min_expected_results - +def test_frontend_search_datasets(frontend_session, query, min_expected_results): + + json = { + "query": """query search($input: SearchInput!) {\n + search(input: $input) {\n + start\n + count\n + total\n + searchResults {\n + entity {\n + ... on Dataset {\n + urn\n + name\n + }\n + }\n + }\n + }\n + }""", + "variables": { + "input": { + "type": "DATASET", + "query": f"{query}", + "start": 0, + "count": 10 + } + } + } -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_frontend_list_users(frontend_session): - response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v1/party/entities") + response = frontend_session.post( + f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + ) response.raise_for_status() - data = response.json() + res_data = response.json() - assert data["status"] == "ok" - assert len(data["userEntities"]) >= 3 + assert res_data + assert res_data["data"] + assert res_data["data"]["search"] + assert res_data["data"]["search"]["total"] >= min_expected_results + assert len(res_data["data"]["search"]["searchResults"]) >= min_expected_results @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_frontend_user_info(frontend_session): - response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v1/user/me") + + urn = f"urn:li:corpuser:datahub" + json = { + "query": """query corpUser($urn: String!) {\n + corpUser(urn: $urn) {\n + urn\n + username\n + editableInfo {\n + pictureLink\n + }\n + info {\n + firstName\n + fullName\n + title\n + email\n + }\n + }\n + }""", + "variables": { + "urn": urn + } + } + response = frontend_session.post( + f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + ) response.raise_for_status() - data = response.json() + res_data = response.json() - assert data["status"] == "ok" - assert data["user"]["userName"] == "datahub" - assert data["user"]["name"] - assert data["user"]["email"] + assert res_data + assert res_data["data"] + assert res_data["data"]["corpUser"] + assert res_data["data"]["corpUser"]["urn"] == urn @pytest.mark.parametrize( @@ -336,28 +387,39 @@ def test_frontend_user_info(frontend_session): ], ) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_frontend_user_info(frontend_session, platform, dataset_name, env): +def test_frontend_datasets(frontend_session, platform, dataset_name, env): urn = f"urn:li:dataset:({platform},{dataset_name},{env})" - + json = { + "query": """query getDataset($urn: String!) {\n + dataset(urn: $urn) {\n + urn\n + name\n + description\n + platform {\n + urn\n + }\n + schemaMetadata {\n + name\n + version\n + createdAt\n + }\n + }\n + }""", + "variables": { + "urn": urn + } + } # Basic dataset info. - response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v2/datasets/{urn}") - response.raise_for_status() - data = response.json() - - assert data["nativeName"] == dataset_name - assert data["fabric"] == env - assert data["uri"] == urn - - # Schema info. - response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v2/datasets/{urn}/schema") + response = frontend_session.post( + f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + ) response.raise_for_status() - data = response.json() - - assert len(data["schema"]["columns"]) >= 2 + res_data = response.json() - # Ownership info. - response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v2/datasets/{urn}/owners") - response.raise_for_status() - data = response.json() + assert res_data + assert res_data["data"] + assert res_data["data"]["dataset"] + assert res_data["data"]["dataset"]["urn"] == urn + assert res_data["data"]["dataset"]["name"] == dataset_name + assert res_data["data"]["dataset"]["platform"]["urn"] == platform - assert len(data["owners"]) >= 1 From 38a5e65bb86ae6380aa2e320d17c34ab2fd9510f Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Wed, 4 Aug 2021 12:29:28 -0700 Subject: [PATCH 4/7] temporarily hide schema tab --- .../dataset/profile/schema/components/SchemaHeader.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx index 1cb896552ea09a..0b64d24cf4c6a9 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx @@ -8,6 +8,7 @@ const SchemaHeaderContainer = styled.div<{ edit?: string }>` ${(props) => (props.edit ? 'justify-content: flex-end; padding-bottom: 16px;' : 'justify-content: space-between;')} `; +// eslint-disable-next-line @typescript-eslint/no-unused-vars const ShowVersionButton = styled(Button)` margin-right: 10px; `; @@ -26,6 +27,7 @@ export default function SchemaHeader({ maxVersion, fetchVersions, editMode, + // eslint-disable-next-line @typescript-eslint/no-unused-vars setEditMode, hasRow, showRaw, @@ -42,12 +44,13 @@ export default function SchemaHeader({ {maxVersion > 0 && !editMode && }
- {maxVersion > 0 && + {/* TODO(Gabe): uncomment when dbt/bigquery flickering has been resolved */} + {/* {maxVersion > 0 && (editMode ? ( setEditMode(false)}>Version History ) : ( setEditMode(true)}>Back - ))} + ))} */} {hasRow && }
From 3ff9ce4c051e1bdc8642b03a28521b76df053547 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Wed, 4 Aug 2021 12:48:10 -0700 Subject: [PATCH 5/7] simplifying hiding logic --- .../dataset/profile/schema/components/SchemaHeader.tsx | 9 ++++----- metadata-ingestion/examples/mce_files/bootstrap_mce.json | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx index 0b64d24cf4c6a9..713547278f23d2 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx @@ -8,9 +8,10 @@ const SchemaHeaderContainer = styled.div<{ edit?: string }>` ${(props) => (props.edit ? 'justify-content: flex-end; padding-bottom: 16px;' : 'justify-content: space-between;')} `; -// eslint-disable-next-line @typescript-eslint/no-unused-vars +// TODO(Gabe): undo display: none when dbt/bigquery flickering has been resolved const ShowVersionButton = styled(Button)` margin-right: 10px; + display: none; `; type Props = { @@ -27,7 +28,6 @@ export default function SchemaHeader({ maxVersion, fetchVersions, editMode, - // eslint-disable-next-line @typescript-eslint/no-unused-vars setEditMode, hasRow, showRaw, @@ -44,13 +44,12 @@ export default function SchemaHeader({ {maxVersion > 0 && !editMode && }
- {/* TODO(Gabe): uncomment when dbt/bigquery flickering has been resolved */} - {/* {maxVersion > 0 && + {maxVersion > 0 && (editMode ? ( setEditMode(false)}>Version History ) : ( setEditMode(true)}>Back - ))} */} + ))} {hasRow && }
diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index c2b732e5c777ba..0b52d701bb80e1 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -1332,7 +1332,7 @@ }, { "com.linkedin.pegasus2avro.common.Status": { - "boolean": false + "removed": false } }, { From 7400f21cebb2cd0ab33a7b6dedf513422d77401f Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Wed, 4 Aug 2021 13:58:45 -0700 Subject: [PATCH 6/7] Revert to upstream --- .../entity/dataset/profile/schema/components/SchemaHeader.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx index 713547278f23d2..1cb896552ea09a 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx @@ -8,10 +8,8 @@ const SchemaHeaderContainer = styled.div<{ edit?: string }>` ${(props) => (props.edit ? 'justify-content: flex-end; padding-bottom: 16px;' : 'justify-content: space-between;')} `; -// TODO(Gabe): undo display: none when dbt/bigquery flickering has been resolved const ShowVersionButton = styled(Button)` margin-right: 10px; - display: none; `; type Props = { From 10f5263d0cc32103c72d0cef4accdbe377602c56 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 6 Aug 2021 10:52:42 -0700 Subject: [PATCH 7/7] Fixing smoke_test broken --- .../src/datahub/metadata/schema.avsc | 12 +++++----- .../src/datahub/metadata/schema_classes.py | 24 ++++++++++++------- .../metadata/schemas/MetadataChangeEvent.avsc | 12 +++++----- .../schemas/MetadataChangeProposal.avsc | 12 +++++----- smoke-test/test_e2e.py | 1 - 5 files changed, 34 insertions(+), 27 deletions(-) diff --git a/metadata-ingestion/src/datahub/metadata/schema.avsc b/metadata-ingestion/src/datahub/metadata/schema.avsc index feae9c9f73d911..c1c038e30d31f7 100644 --- a/metadata-ingestion/src/datahub/metadata/schema.avsc +++ b/metadata-ingestion/src/datahub/metadata/schema.avsc @@ -5219,20 +5219,20 @@ "fields": [ { "type": [ - "null", - "long" + "long", + "null" ], "name": "lastObserved", - "default": null, + "default": 0, "doc": "The timestamp the metadata was observed at" }, { "type": [ - "null", - "string" + "string", + "null" ], "name": "runId", - "default": null, + "default": "no-run-id-provided", "doc": "The run id that produced the metadata" }, { diff --git a/metadata-ingestion/src/datahub/metadata/schema_classes.py b/metadata-ingestion/src/datahub/metadata/schema_classes.py index 1b1e533fc94a03..b211a36bb3fe00 100644 --- a/metadata-ingestion/src/datahub/metadata/schema_classes.py +++ b/metadata-ingestion/src/datahub/metadata/schema_classes.py @@ -7381,14 +7381,22 @@ class SystemMetadataClass(DictWrapper): RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.mxe.SystemMetadata") def __init__(self, - lastObserved: Union[None, int]=None, - runId: Union[None, str]=None, + lastObserved: Optional[Union[int, None]]=None, + runId: Optional[Union[str, None]]=None, properties: Union[None, Dict[str, str]]=None, ): super().__init__() - self.lastObserved = lastObserved - self.runId = runId + if lastObserved is None: + # default: 0 + self.lastObserved = self.RECORD_SCHEMA.field_map["lastObserved"].default + else: + self.lastObserved = lastObserved + if runId is None: + # default: 'no-run-id-provided' + self.runId = self.RECORD_SCHEMA.field_map["runId"].default + else: + self.runId = runId self.properties = properties @classmethod @@ -7405,23 +7413,23 @@ def _restore_defaults(self) -> None: @property - def lastObserved(self) -> Union[None, int]: + def lastObserved(self) -> Union[int, None]: """Getter: The timestamp the metadata was observed at""" return self._inner_dict.get('lastObserved') # type: ignore @lastObserved.setter - def lastObserved(self, value: Union[None, int]) -> None: + def lastObserved(self, value: Union[int, None]) -> None: """Setter: The timestamp the metadata was observed at""" self._inner_dict['lastObserved'] = value @property - def runId(self) -> Union[None, str]: + def runId(self) -> Union[str, None]: """Getter: The run id that produced the metadata""" return self._inner_dict.get('runId') # type: ignore @runId.setter - def runId(self, value: Union[None, str]) -> None: + def runId(self, value: Union[str, None]) -> None: """Setter: The run id that produced the metadata""" self._inner_dict['runId'] = value diff --git a/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeEvent.avsc b/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeEvent.avsc index 17ae20c4b62fa6..e8bcce446063ed 100644 --- a/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeEvent.avsc +++ b/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeEvent.avsc @@ -5163,20 +5163,20 @@ { "name": "lastObserved", "type": [ - "null", - "long" + "long", + "null" ], "doc": "The timestamp the metadata was observed at", - "default": null + "default": 0 }, { "name": "runId", "type": [ - "null", - "string" + "string", + "null" ], "doc": "The run id that produced the metadata", - "default": null + "default": "no-run-id-provided" }, { "name": "properties", diff --git a/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeProposal.avsc b/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeProposal.avsc index cadf7e13195435..f1bd5328cd131d 100644 --- a/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeProposal.avsc +++ b/metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeProposal.avsc @@ -185,20 +185,20 @@ { "name": "lastObserved", "type": [ - "null", - "long" + "long", + "null" ], "doc": "The timestamp the metadata was observed at", - "default": null + "default": 0 }, { "name": "runId", "type": [ - "null", - "string" + "string", + "null" ], "doc": "The run id that produced the metadata", - "default": null + "default": "no-run-id-provided" }, { "name": "properties", diff --git a/smoke-test/test_e2e.py b/smoke-test/test_e2e.py index 79bc93bc2a8da2..d38c174afb43d8 100644 --- a/smoke-test/test_e2e.py +++ b/smoke-test/test_e2e.py @@ -422,7 +422,6 @@ def test_frontend_datasets(frontend_session, platform, dataset_name, env): assert res_data["data"]["dataset"]["urn"] == urn assert res_data["data"]["dataset"]["name"] == dataset_name assert res_data["data"]["dataset"]["platform"]["urn"] == platform - assert len(data["owners"]) >= 1 @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_ingest_with_system_metadata():