diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java index 7a41a42af77c25..800ce4218ed1f7 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java @@ -15,6 +15,7 @@ import com.linkedin.identity.CorpUserEditableInfo; import com.linkedin.identity.CorpUserInfo; import com.linkedin.identity.GroupMembership; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.CorpGroupAspect; import com.linkedin.metadata.aspect.CorpGroupAspectArray; import com.linkedin.metadata.aspect.CorpUserAspect; @@ -64,7 +65,7 @@ @Slf4j public class OidcCallbackLogic extends DefaultCallbackLogic { - private static final String SYSTEM_PRINCIPAL = "urn:li:corpuser:system"; + private static final String SYSTEM_ACTOR = Constants.SYSTEM_ACTOR; private final EntityClient _entityClient = GmsClientFactory.getEntitiesClient(); private final SsoManager _ssoManager; @@ -271,7 +272,7 @@ private void tryProvisionUser(CorpUserSnapshot corpUserSnapshot) { // 1. Check if this user already exists. try { - final Entity corpUser = _entityClient.get(corpUserSnapshot.getUrn(), SYSTEM_PRINCIPAL); + final Entity corpUser = _entityClient.get(corpUserSnapshot.getUrn(), SYSTEM_ACTOR); log.debug(String.format("Fetched GMS user with urn %s",corpUserSnapshot.getUrn())); @@ -281,7 +282,7 @@ private void tryProvisionUser(CorpUserSnapshot corpUserSnapshot) { // 2. The user does not exist. Provision them. final Entity newEntity = new Entity(); newEntity.setValue(Snapshot.create(corpUserSnapshot)); - _entityClient.update(newEntity, SYSTEM_PRINCIPAL); + _entityClient.update(newEntity, SYSTEM_ACTOR); log.debug(String.format("Successfully provisioned user %s", corpUserSnapshot.getUrn())); } @@ -302,7 +303,7 @@ private void tryProvisionGroups(List corpGroups) { // 1. Check if this user already exists. try { final Set urnsToFetch = corpGroups.stream().map(CorpGroupSnapshot::getUrn).collect(Collectors.toSet()); - final Map existingGroups = _entityClient.batchGet(urnsToFetch, SYSTEM_PRINCIPAL); + final Map existingGroups = _entityClient.batchGet(urnsToFetch, SYSTEM_ACTOR); log.debug(String.format("Fetched GMS groups with urns %s", existingGroups.keySet())); @@ -335,7 +336,7 @@ private void tryProvisionGroups(List corpGroups) { // Now batch create all entities identified to create. _entityClient.batchUpdate(groupsToCreate.stream().map(groupSnapshot -> new Entity().setValue(Snapshot.create(groupSnapshot)) - ).collect(Collectors.toSet()), SYSTEM_PRINCIPAL); + ).collect(Collectors.toSet()), SYSTEM_ACTOR); log.debug(String.format("Successfully provisioned groups with urns %s", groupsToCreateUrns)); @@ -349,7 +350,7 @@ private void tryProvisionGroups(List corpGroups) { private void verifyPreProvisionedUser(CorpuserUrn urn) { // Validate that the user exists in the system (there is more than just a key aspect for them, as of today). try { - final Entity corpUser = _entityClient.get(urn, SYSTEM_PRINCIPAL); + final Entity corpUser = _entityClient.get(urn, SYSTEM_ACTOR); log.debug(String.format("Fetched GMS user with urn %s", urn)); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java index b1c046bd601eb3..dc3b791616cd3b 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java @@ -8,6 +8,7 @@ import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.entity.EntityService; @@ -149,7 +150,7 @@ public Function executable() { browsePaths = BrowsePathUtils.buildBrowsePath(urn); final AuditStamp browsePathsStamp = new AuditStamp(); - browsePathsStamp.setActor(Urn.createFromString("urn:li:principal:system")); + browsePathsStamp.setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)); browsePathsStamp.setTime(System.currentTimeMillis()); _entityService.ingestAspect(urn, BROWSE_PATHS_ASPECT_NAME, browsePaths, browsePathsStamp); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/IngestDataPlatformsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/IngestDataPlatformsStep.java deleted file mode 100644 index ad18d49d5bf79a..00000000000000 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/IngestDataPlatformsStep.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.linkedin.datahub.upgrade.nocode; - -import com.linkedin.common.AuditStamp; -import com.linkedin.common.urn.DataPlatformUrn; -import com.linkedin.common.urn.Urn; -import com.linkedin.datahub.upgrade.UpgradeContext; -import com.linkedin.datahub.upgrade.UpgradeStep; -import com.linkedin.datahub.upgrade.UpgradeStepResult; -import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; -import com.linkedin.dataplatform.DataPlatformInfo; -import com.linkedin.metadata.utils.PegasusUtils; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.resources.dataplatform.utils.DataPlatformsUtil; -import java.net.URISyntaxException; -import java.time.Clock; -import java.util.Map; -import java.util.function.Function; - - -public class IngestDataPlatformsStep implements UpgradeStep { - - private final EntityService _entityService; - - public IngestDataPlatformsStep(final EntityService entityService) { - _entityService = entityService; - } - - @Override - public String id() { - return "IngestDataPlatformsStep"; - } - - @Override - public int retryCount() { - return 2; - } - - @Override - public Function executable() { - return (context) -> { - - context.report().addLine("Preparing to ingest DataPlatforms..."); - - Map urnToInfo = DataPlatformsUtil.getDataPlatformInfoMap(); - - context.report().addLine(String.format("Found %s DataPlatforms", urnToInfo.keySet().size())); - - for (final Map.Entry entry : urnToInfo.entrySet()) { - AuditStamp auditStamp; - try { - auditStamp = new AuditStamp().setActor(Urn.createFromString("urn:li:principal:system")).setTime( - Clock.systemUTC().millis()); - } catch (URISyntaxException e) { - throw new RuntimeException("Failed to create Actor Urn"); - } - - _entityService.ingestAspect( - entry.getKey(), - PegasusUtils.getAspectNameFromSchema(entry.getValue().schema()), - entry.getValue(), - auditStamp - ); - } - - context.report().addLine(String.format("Successfully ingested %s DataPlatforms.", urnToInfo.keySet().size())); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); - }; - } -} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java index 793b7f6db75a8b..705e8b9dbfbfcd 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java @@ -66,7 +66,6 @@ private List buildUpgradeSteps( steps.add(new GMSQualificationStep()); steps.add(new UpgradeQualificationStep(server)); steps.add(new CreateAspectTableStep(server)); - steps.add(new IngestDataPlatformsStep(entityService)); steps.add(new DataMigrationStep(server, entityService, entityRegistry)); steps.add(new GMSEnableWriteModeStep(entityClient)); return steps; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java index 0912cbb4d73010..258b331e5108d5 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java @@ -61,7 +61,7 @@ private boolean isQualified(EbeanServer server, UpgradeContext context) { boolean v2TableExists = AspectStorageValidationUtil.checkV2TableExists(server); if (v2TableExists) { context.report().addLine("-- V2 table exists"); - long v2TableRowCount = AspectStorageValidationUtil.getV2RowCount(server); + long v2TableRowCount = AspectStorageValidationUtil.getV2NonSystemRowCount(server); if (v2TableRowCount == 0) { context.report().addLine("-- V2 table is empty"); return true; diff --git a/docker/mariadb/init.sql b/docker/mariadb/init.sql index 95aa80dd3070d0..04932b172f8920 100644 --- a/docker/mariadb/init.sql +++ b/docker/mariadb/init.sql @@ -17,12 +17,12 @@ insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); diff --git a/docker/mysql-setup/init.sql b/docker/mysql-setup/init.sql index cc6e934d66e035..83ae2e0447a5a7 100644 --- a/docker/mysql-setup/init.sql +++ b/docker/mysql-setup/init.sql @@ -23,14 +23,14 @@ INSERT INTO temp_metadata_aspect_v2 (urn, aspect, version, metadata, createdon, 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- only add default records if metadata_aspect is empty INSERT INTO metadata_aspect_v2 diff --git a/docker/mysql/init.sql b/docker/mysql/init.sql index 1d1bfe3f39787a..6d08983e5994ce 100644 --- a/docker/mysql/init.sql +++ b/docker/mysql/init.sql @@ -17,14 +17,14 @@ INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- create metadata index table diff --git a/docker/postgres-setup/init.sql b/docker/postgres-setup/init.sql index 721d55ca36fc36..26020359cf98af 100644 --- a/docker/postgres-setup/init.sql +++ b/docker/postgres-setup/init.sql @@ -19,14 +19,14 @@ INSERT INTO temp_metadata_aspect_v2 (urn, aspect, version, metadata, createdon, 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- only add default records if metadata_aspect is empty INSERT INTO metadata_aspect_v2 diff --git a/docker/postgres/init.sql b/docker/postgres/init.sql index 5c5216181c5f95..162422e25fbc47 100644 --- a/docker/postgres/init.sql +++ b/docker/postgres/init.sql @@ -17,12 +17,12 @@ insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); diff --git a/docker/quickstart/mysql/init.sql b/docker/quickstart/mysql/init.sql index 0f8ec87d0187ae..d94e6bd9b75e43 100644 --- a/docker/quickstart/mysql/init.sql +++ b/docker/quickstart/mysql/init.sql @@ -17,14 +17,14 @@ INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- create metadata index table diff --git a/metadata-ingestion/tests/integration/mysql/setup/setup.sql b/metadata-ingestion/tests/integration/mysql/setup/setup.sql index f9cdfbb55bf982..f86c002e27c7d0 100644 --- a/metadata-ingestion/tests/integration/mysql/setup/setup.sql +++ b/metadata-ingestion/tests/integration/mysql/setup/setup.sql @@ -19,14 +19,14 @@ insert into metadata_aspect (urn, aspect, version, metadata, createdon, createdb 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'com.linkedin.identity.CorpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- create metadata index table diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java index 823c515bbbbdac..568af4ab0e94b8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.ebean.EbeanAspectV1; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import io.ebean.EbeanServer; @@ -7,6 +8,8 @@ import io.ebean.SqlRow; import java.util.List; +import static io.ebean.Expr.*; + public class AspectStorageValidationUtil { @@ -18,8 +21,11 @@ public static long getV1RowCount(EbeanServer server) { return server.find(EbeanAspectV1.class).findCount(); } - public static long getV2RowCount(EbeanServer server) { - return server.find(EbeanAspectV2.class).findCount(); + /** + * Get the number of rows created not by the DataHub system actor (urn:li:corpuser:__datahub_system) + */ + public static long getV2NonSystemRowCount(EbeanServer server) { + return server.find(EbeanAspectV2.class).where(ne("createdby", Constants.SYSTEM_ACTOR)).findCount(); } public static boolean checkV2TableExists(EbeanServer server) { diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java index 86227d604045b4..3d9e63ad10cd06 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java @@ -57,7 +57,7 @@ public void execute() throws IOException, URISyntaxException { RecordUtils.toRecordTemplate(DataPlatformInfo.class, dataPlatform.get("aspect").toString()); final AuditStamp aspectAuditStamp = - new AuditStamp().setActor(Urn.createFromString(Constants.UNKNOWN_ACTOR)).setTime(System.currentTimeMillis()); + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); _entityService.ingestAspect(urn, PLATFORM_ASPECT_NAME, info, aspectAuditStamp); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java index 5cc4ae6d5ab7cf..f4e4010b8eb651 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java @@ -5,6 +5,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.boot.BootstrapStep; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.dao.utils.RecordUtils; @@ -92,7 +93,7 @@ private void ingestPolicy(final Urn urn, final DataHubPolicyInfo info) throws UR keyAspectProposal.setEntityUrn(urn); _entityService.ingestProposal(keyAspectProposal, - new AuditStamp().setActor(Urn.createFromString("urn:li:corpuser:system")).setTime(System.currentTimeMillis())); + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); final MetadataChangeProposal proposal = new MetadataChangeProposal(); proposal.setEntityUrn(urn); @@ -102,7 +103,7 @@ private void ingestPolicy(final Urn urn, final DataHubPolicyInfo info) throws UR proposal.setChangeType(ChangeType.UPSERT); _entityService.ingestProposal(proposal, - new AuditStamp().setActor(Urn.createFromString("urn:li:corpuser:system")).setTime(System.currentTimeMillis())); + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); } private boolean hasDefaultPolicies() throws URISyntaxException { diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java index dd3a536dd0732a..5cb03900366244 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -6,7 +6,7 @@ public class Constants { public static final String ACTOR_HEADER_NAME = "X-DataHub-Actor"; public static final String DATAHUB_ACTOR = "urn:li:corpuser:datahub"; // Super user. - public static final String SYSTEM_ACTOR = "urn:li:principal:datahub"; // DataHub internal service principal. + public static final String SYSTEM_ACTOR = "urn:li:corpuser:__datahub_system"; // DataHub internal service principal. public static final String UNKNOWN_ACTOR = "urn:li:corpuser:UNKNOWN"; // Unknown principal. public static final Long ASPECT_LATEST_VERSION = 0L;