From dcaddf7f9fe9d55c5bfae9d1e586539229a1ac99 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 26 Sep 2024 22:09:47 -0500 Subject: [PATCH 1/5] chore(cleanup): remove legacy bootstrap step --- .../boot/MCLBootstrapManagerFactory.java | 4 - .../boot/MCPBootstrapManagerFactory.java | 4 - .../src/main/resources/application.yaml | 6 - .../factories/BootstrapManagerFactory.java | 16 - .../boot/steps/BackfillBrowsePathsV2Step.java | 156 ------- .../steps/UpgradeDefaultBrowsePathsStep.java | 145 ------- .../steps/BackfillBrowsePathsV2StepTest.java | 188 -------- .../UpgradeDefaultBrowsePathsStepTest.java | 406 ------------------ 8 files changed, 925 deletions(-) delete mode 100644 metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java delete mode 100644 metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java delete mode 100644 metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java delete mode 100644 metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCLBootstrapManagerFactory.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCLBootstrapManagerFactory.java index 8ad1638115dae..8a913910b3523 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCLBootstrapManagerFactory.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCLBootstrapManagerFactory.java @@ -11,7 +11,6 @@ import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; @@ -27,9 +26,6 @@ public class MCLBootstrapManagerFactory { @Autowired private ConfigurationProvider _configurationProvider; - @Value("${bootstrap.upgradeDefaultBrowsePaths.enabled}") - private Boolean _upgradeDefaultBrowsePathsEnabled; - @Bean(name = "mclBootstrapManager") @Scope("singleton") @Nonnull diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCPBootstrapManagerFactory.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCPBootstrapManagerFactory.java index 0220764cd99d6..5419fa48ee5ba 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCPBootstrapManagerFactory.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/boot/MCPBootstrapManagerFactory.java @@ -11,7 +11,6 @@ import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; @@ -27,9 +26,6 @@ public class MCPBootstrapManagerFactory { @Autowired private ConfigurationProvider _configurationProvider; - @Value("${bootstrap.upgradeDefaultBrowsePaths.enabled}") - private Boolean _upgradeDefaultBrowsePathsEnabled; - @Bean(name = "mcpBootstrapManager") @Scope("singleton") @Nonnull diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 0ce0b976c976e..5e07bfc479e93 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -343,12 +343,6 @@ incidents: consumerGroupSuffix: ${INCIDENTS_HOOK_CONSUMER_GROUP_SUFFIX:} bootstrap: - upgradeDefaultBrowsePaths: - enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones - backfillBrowsePathsV2: - enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate - reprocessDefaultBrowsePathsV2: - enabled: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]} policies: file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json} # eg for local file diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java index 97a009dcbbb6d..9e29883f439a7 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java @@ -9,7 +9,6 @@ import com.linkedin.metadata.boot.BootstrapManager; import com.linkedin.metadata.boot.BootstrapStep; import com.linkedin.metadata.boot.dependencies.BootstrapDependency; -import com.linkedin.metadata.boot.steps.BackfillBrowsePathsV2Step; import com.linkedin.metadata.boot.steps.IndexDataPlatformsStep; import com.linkedin.metadata.boot.steps.IngestDataPlatformInstancesStep; import com.linkedin.metadata.boot.steps.IngestDataPlatformsStep; @@ -25,7 +24,6 @@ import com.linkedin.metadata.boot.steps.RestoreColumnLineageIndices; import com.linkedin.metadata.boot.steps.RestoreDbtSiblingsIndices; import com.linkedin.metadata.boot.steps.RestoreGlossaryIndices; -import com.linkedin.metadata.boot.steps.UpgradeDefaultBrowsePathsStep; import com.linkedin.metadata.boot.steps.WaitForSystemUpdateStep; import com.linkedin.metadata.entity.AspectMigrationsDao; import com.linkedin.metadata.entity.EntityService; @@ -89,12 +87,6 @@ public class BootstrapManagerFactory { @Autowired private ConfigurationProvider _configurationProvider; - @Value("${bootstrap.upgradeDefaultBrowsePaths.enabled}") - private Boolean _upgradeDefaultBrowsePathsEnabled; - - @Value("${bootstrap.backfillBrowsePathsV2.enabled}") - private Boolean _backfillBrowsePathsV2Enabled; - @Value("${bootstrap.policies.file}") private Resource _policiesResource; @@ -154,14 +146,6 @@ protected BootstrapManager createInstance( ingestDataTypesStep, ingestEntityTypesStep)); - if (_upgradeDefaultBrowsePathsEnabled) { - finalSteps.add(new UpgradeDefaultBrowsePathsStep(_entityService)); - } - - if (_backfillBrowsePathsV2Enabled) { - finalSteps.add(new BackfillBrowsePathsV2Step(_entityService, _searchService)); - } - return new BootstrapManager(finalSteps); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java deleted file mode 100644 index 2c00c73c96549..0000000000000 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java +++ /dev/null @@ -1,156 +0,0 @@ -package com.linkedin.metadata.boot.steps; - -import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion; -import static com.linkedin.metadata.utils.CriterionUtils.buildIsNullCriterion; -import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.linkedin.common.AuditStamp; -import com.linkedin.common.BrowsePathsV2; -import com.linkedin.common.urn.Urn; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil; -import com.linkedin.metadata.boot.UpgradeStep; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.query.filter.ConjunctiveCriterion; -import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; -import com.linkedin.metadata.query.filter.Criterion; -import com.linkedin.metadata.query.filter.CriterionArray; -import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.search.ScrollResult; -import com.linkedin.metadata.search.SearchEntity; -import com.linkedin.metadata.search.SearchService; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; -import io.datahubproject.metadata.context.OperationContext; -import java.util.Set; -import javax.annotation.Nonnull; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class BackfillBrowsePathsV2Step extends UpgradeStep { - - private static final Set ENTITY_TYPES_TO_MIGRATE = - ImmutableSet.of( - Constants.DATASET_ENTITY_NAME, - Constants.DASHBOARD_ENTITY_NAME, - Constants.CHART_ENTITY_NAME, - Constants.DATA_JOB_ENTITY_NAME, - Constants.DATA_FLOW_ENTITY_NAME, - Constants.ML_MODEL_ENTITY_NAME, - Constants.ML_MODEL_GROUP_ENTITY_NAME, - Constants.ML_FEATURE_TABLE_ENTITY_NAME, - Constants.ML_FEATURE_ENTITY_NAME); - private static final String VERSION = "2"; - private static final String UPGRADE_ID = "backfill-default-browse-paths-v2-step"; - private static final Integer BATCH_SIZE = 5000; - - private final SearchService searchService; - - public BackfillBrowsePathsV2Step(EntityService entityService, SearchService searchService) { - super(entityService, VERSION, UPGRADE_ID); - this.searchService = searchService; - } - - @Nonnull - @Override - public ExecutionMode getExecutionMode() { - return ExecutionMode.BLOCKING; // ensure there are no write conflicts. - } - - @Override - public void upgrade(@Nonnull OperationContext systemOpContext) throws Exception { - final AuditStamp auditStamp = - new AuditStamp() - .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)) - .setTime(System.currentTimeMillis()); - - String scrollId = null; - for (String entityType : ENTITY_TYPES_TO_MIGRATE) { - int migratedCount = 0; - do { - log.info( - String.format( - "Upgrading batch %s-%s of browse paths for entity type %s", - migratedCount, migratedCount + BATCH_SIZE, entityType)); - scrollId = backfillBrowsePathsV2(systemOpContext, entityType, auditStamp, scrollId); - migratedCount += BATCH_SIZE; - } while (scrollId != null); - } - } - - private String backfillBrowsePathsV2( - @Nonnull OperationContext systemOperationContext, - String entityType, - AuditStamp auditStamp, - String scrollId) - throws Exception { - - // Condition: has `browsePaths` AND does NOT have `browsePathV2` - Criterion missingBrowsePathV2 = buildIsNullCriterion("browsePathV2"); - - // Excludes entities without browsePaths - Criterion hasBrowsePathV1 = buildExistsCriterion("browsePaths"); - - CriterionArray criterionArray = new CriterionArray(); - criterionArray.add(missingBrowsePathV2); - criterionArray.add(hasBrowsePathV1); - - ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion(); - conjunctiveCriterion.setAnd(criterionArray); - - ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray(); - conjunctiveCriterionArray.add(conjunctiveCriterion); - - Filter filter = new Filter(); - filter.setOr(conjunctiveCriterionArray); - - final ScrollResult scrollResult = - searchService.scrollAcrossEntities( - systemOperationContext, - ImmutableList.of(entityType), - "*", - filter, - null, - scrollId, - "5m", - BATCH_SIZE); - if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) { - return null; - } - - for (SearchEntity searchEntity : scrollResult.getEntities()) { - try { - ingestBrowsePathsV2(systemOperationContext, searchEntity.getEntity(), auditStamp); - } catch (Exception e) { - // don't stop the whole step because of one bad urn or one bad ingestion - log.error( - String.format( - "Error ingesting default browsePathsV2 aspect for urn %s", - searchEntity.getEntity()), - e); - } - } - - return scrollResult.getScrollId(); - } - - private void ingestBrowsePathsV2( - @Nonnull OperationContext systemOperationContext, Urn urn, AuditStamp auditStamp) - throws Exception { - BrowsePathsV2 browsePathsV2 = - DefaultAspectsUtil.buildDefaultBrowsePathV2( - systemOperationContext, urn, true, entityService); - log.debug(String.format("Adding browse path v2 for urn %s with value %s", urn, browsePathsV2)); - MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(urn); - proposal.setEntityType(urn.getEntityType()); - proposal.setAspectName(Constants.BROWSE_PATHS_V2_ASPECT_NAME); - proposal.setChangeType(ChangeType.UPSERT); - proposal.setSystemMetadata(createDefaultSystemMetadata()); - proposal.setAspect(GenericRecordUtils.serializeAspect(browsePathsV2)); - entityService.ingestProposal(systemOperationContext, proposal, auditStamp, false); - } -} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java deleted file mode 100644 index 89846476a9875..0000000000000 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java +++ /dev/null @@ -1,145 +0,0 @@ -package com.linkedin.metadata.boot.steps; - -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; - -import com.google.common.collect.ImmutableSet; -import com.linkedin.common.AuditStamp; -import com.linkedin.common.BrowsePaths; -import com.linkedin.common.urn.Urn; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil; -import com.linkedin.metadata.boot.UpgradeStep; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.ListResult; -import com.linkedin.metadata.query.ExtraInfo; -import com.linkedin.metadata.search.utils.BrowsePathUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; -import io.datahubproject.metadata.context.OperationContext; -import java.util.Set; -import javax.annotation.Nonnull; -import lombok.extern.slf4j.Slf4j; - -/** - * This is an opt-in optional upgrade step to migrate your browse paths to the new truncated form. - * It is idempotent, can be retried as many times as necessary. - */ -@Slf4j -public class UpgradeDefaultBrowsePathsStep extends UpgradeStep { - - private static final Set ENTITY_TYPES_TO_MIGRATE = - ImmutableSet.of( - Constants.DATASET_ENTITY_NAME, - Constants.DASHBOARD_ENTITY_NAME, - Constants.CHART_ENTITY_NAME, - Constants.DATA_JOB_ENTITY_NAME, - Constants.DATA_FLOW_ENTITY_NAME); - private static final String VERSION = "1"; - private static final String UPGRADE_ID = "upgrade-default-browse-paths-step"; - private static final Integer BATCH_SIZE = 5000; - - public UpgradeDefaultBrowsePathsStep(EntityService entityService) { - super(entityService, VERSION, UPGRADE_ID); - } - - @Override - public void upgrade(@Nonnull OperationContext systemOperationContext) throws Exception { - final AuditStamp auditStamp = - new AuditStamp() - .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)) - .setTime(System.currentTimeMillis()); - - int total = 0; - for (String entityType : ENTITY_TYPES_TO_MIGRATE) { - int migratedCount = 0; - do { - log.info( - String.format( - "Upgrading batch %s-%s out of %s of browse paths for entity type %s", - migratedCount, migratedCount + BATCH_SIZE, total, entityType)); - total = - getAndMigrateBrowsePaths(systemOperationContext, entityType, migratedCount, auditStamp); - migratedCount += BATCH_SIZE; - } while (migratedCount < total); - } - log.info("Successfully upgraded all browse paths!"); - } - - @Nonnull - @Override - public ExecutionMode getExecutionMode() { - return ExecutionMode.BLOCKING; // ensure there are no write conflicts. - } - - private int getAndMigrateBrowsePaths( - @Nonnull OperationContext opContext, String entityType, int start, AuditStamp auditStamp) - throws Exception { - - final ListResult latestAspects = - entityService.listLatestAspects( - opContext, entityType, Constants.BROWSE_PATHS_ASPECT_NAME, start, BATCH_SIZE); - - if (latestAspects.getTotalCount() == 0 - || latestAspects.getValues() == null - || latestAspects.getMetadata() == null) { - log.debug( - String.format( - "Found 0 browse paths for entity with type %s. Skipping migration!", entityType)); - return 0; - } - - if (latestAspects.getValues().size() != latestAspects.getMetadata().getExtraInfos().size()) { - // Bad result -- we should log that we cannot migrate this batch of paths. - log.warn( - "Failed to match browse path aspects with corresponding urns. Found mismatched length between aspects ({})" - + "and metadata ({}) for metadata {}", - latestAspects.getValues().size(), - latestAspects.getMetadata().getExtraInfos().size(), - latestAspects.getMetadata()); - return latestAspects.getTotalCount(); - } - - for (int i = 0; i < latestAspects.getValues().size(); i++) { - - ExtraInfo info = latestAspects.getMetadata().getExtraInfos().get(i); - RecordTemplate browsePathsRec = latestAspects.getValues().get(i); - - // Assert on 2 conditions: - // 1. The latest browse path aspect contains only 1 browse path - // 2. The latest browse path matches exactly the legacy default path - - Urn urn = info.getUrn(); - BrowsePaths browsePaths = (BrowsePaths) browsePathsRec; - - log.debug(String.format("Inspecting browse path for urn %s, value %s", urn, browsePaths)); - - if (browsePaths.hasPaths() && browsePaths.getPaths().size() == 1) { - String legacyBrowsePath = - BrowsePathUtils.getLegacyDefaultBrowsePath(urn, opContext.getEntityRegistry()); - log.debug(String.format("Legacy browse path for urn %s, value %s", urn, legacyBrowsePath)); - if (legacyBrowsePath.equals(browsePaths.getPaths().get(0))) { - migrateBrowsePath(opContext, urn, auditStamp); - } - } - } - - return latestAspects.getTotalCount(); - } - - private void migrateBrowsePath( - @Nonnull OperationContext opContext, Urn urn, AuditStamp auditStamp) throws Exception { - BrowsePaths newPaths = DefaultAspectsUtil.buildDefaultBrowsePath(opContext, urn, entityService); - log.debug(String.format("Updating browse path for urn %s to value %s", urn, newPaths)); - MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(urn); - proposal.setEntityType(urn.getEntityType()); - proposal.setAspectName(Constants.BROWSE_PATHS_ASPECT_NAME); - proposal.setChangeType(ChangeType.UPSERT); - proposal.setSystemMetadata(createDefaultSystemMetadata()); - proposal.setAspect(GenericRecordUtils.serializeAspect(newPaths)); - entityService.ingestProposal(opContext, proposal, auditStamp, false); - } -} diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java deleted file mode 100644 index 54c2cd73dfe1d..0000000000000 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java +++ /dev/null @@ -1,188 +0,0 @@ -package com.linkedin.metadata.boot.steps; - -import static com.linkedin.metadata.Constants.CONTAINER_ASPECT_NAME; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; - -import com.google.common.collect.ImmutableList; -import com.linkedin.common.AuditStamp; -import com.linkedin.common.urn.Urn; -import com.linkedin.common.urn.UrnUtils; -import com.linkedin.entity.Aspect; -import com.linkedin.entity.EntityResponse; -import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.entity.EnvelopedAspectMap; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.search.ScrollResult; -import com.linkedin.metadata.search.SearchEntity; -import com.linkedin.metadata.search.SearchEntityArray; -import com.linkedin.metadata.search.SearchService; -import com.linkedin.mxe.MetadataChangeProposal; -import io.datahubproject.metadata.context.OperationContext; -import io.datahubproject.test.metadata.context.TestOperationContexts; -import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.mockito.Mockito; -import org.testng.annotations.Test; - -public class BackfillBrowsePathsV2StepTest { - private static final String VERSION = "2"; - private static final String UPGRADE_URN = - String.format( - "urn:li:%s:%s", - Constants.DATA_HUB_UPGRADE_ENTITY_NAME, "backfill-default-browse-paths-v2-step"); - - private static final String DATASET_URN = - "urn:li:dataset:(urn:li:dataPlatform:platform,name,PROD)"; - private static final String DASHBOARD_URN = "urn:li:dashboard:(airflow,id)"; - private static final String CHART_URN = "urn:li:chart:(looker,baz)"; - private static final String DATA_JOB_URN = - "urn:li:dataJob:(urn:li:dataFlow:(airflow,test,prod),test1)"; - private static final String DATA_FLOW_URN = "urn:li:dataFlow:(orchestrator,flowId,cluster)"; - private static final String ML_MODEL_URN = - "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,trustmodel,PROD)"; - private static final String ML_MODEL_GROUP_URN = - "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)"; - private static final String ML_FEATURE_TABLE_URN = - "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)"; - private static final String ML_FEATURE_URN = "urn:li:mlFeature:(test,feature_1)"; - private static final List ENTITY_TYPES = - ImmutableList.of( - Constants.DATASET_ENTITY_NAME, - Constants.DASHBOARD_ENTITY_NAME, - Constants.CHART_ENTITY_NAME, - Constants.DATA_JOB_ENTITY_NAME, - Constants.DATA_FLOW_ENTITY_NAME, - Constants.ML_MODEL_ENTITY_NAME, - Constants.ML_MODEL_GROUP_ENTITY_NAME, - Constants.ML_FEATURE_TABLE_ENTITY_NAME, - Constants.ML_FEATURE_ENTITY_NAME); - private static final List ENTITY_URNS = - ImmutableList.of( - UrnUtils.getUrn(DATASET_URN), - UrnUtils.getUrn(DASHBOARD_URN), - UrnUtils.getUrn(CHART_URN), - UrnUtils.getUrn(DATA_JOB_URN), - UrnUtils.getUrn(DATA_FLOW_URN), - UrnUtils.getUrn(ML_MODEL_URN), - UrnUtils.getUrn(ML_MODEL_GROUP_URN), - UrnUtils.getUrn(ML_FEATURE_TABLE_URN), - UrnUtils.getUrn(ML_FEATURE_URN)); - - @Test - public void testExecuteNoExistingBrowsePaths() throws Exception { - final EntityService mockService = initMockService(); - final SearchService mockSearchService = initMockSearchService(); - - final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); - Mockito.when( - mockService.getEntityV2( - any(OperationContext.class), - Mockito.eq(Constants.DATA_HUB_UPGRADE_ENTITY_NAME), - Mockito.eq(upgradeEntityUrn), - Mockito.eq(Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)))) - .thenReturn(null); - - BackfillBrowsePathsV2Step backfillBrowsePathsV2Step = - new BackfillBrowsePathsV2Step(mockService, mockSearchService); - backfillBrowsePathsV2Step.execute(TestOperationContexts.systemContextNoSearchAuthorization()); - - Mockito.verify(mockSearchService, Mockito.times(9)) - .scrollAcrossEntities( - any(OperationContext.class), - any(), - Mockito.eq("*"), - any(Filter.class), - Mockito.eq(null), - Mockito.eq(null), - Mockito.eq("5m"), - Mockito.eq(5000)); - // Verify that 11 aspects are ingested, 2 for the upgrade request / result, 9 for ingesting 1 of - // each entity type - Mockito.verify(mockService, Mockito.times(11)) - .ingestProposal( - any(OperationContext.class), - any(MetadataChangeProposal.class), - any(), - Mockito.eq(false)); - } - - @Test - public void testDoesNotRunWhenAlreadyExecuted() throws Exception { - final EntityService mockService = mock(EntityService.class); - final SearchService mockSearchService = initMockSearchService(); - - final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); - com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest = - new com.linkedin.upgrade.DataHubUpgradeRequest().setVersion(VERSION); - Map upgradeRequestAspects = new HashMap<>(); - upgradeRequestAspects.put( - Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME, - new EnvelopedAspect().setValue(new Aspect(upgradeRequest.data()))); - EntityResponse response = - new EntityResponse().setAspects(new EnvelopedAspectMap(upgradeRequestAspects)); - Mockito.when( - mockService.getEntityV2( - any(OperationContext.class), - Mockito.eq(Constants.DATA_HUB_UPGRADE_ENTITY_NAME), - Mockito.eq(upgradeEntityUrn), - Mockito.eq(Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)))) - .thenReturn(response); - - BackfillBrowsePathsV2Step backfillBrowsePathsV2Step = - new BackfillBrowsePathsV2Step(mockService, mockSearchService); - backfillBrowsePathsV2Step.execute(mock(OperationContext.class)); - - Mockito.verify(mockService, Mockito.times(0)) - .ingestProposal( - any(OperationContext.class), - any(MetadataChangeProposal.class), - any(AuditStamp.class), - Mockito.anyBoolean()); - } - - private EntityService initMockService() throws URISyntaxException { - final EntityService mockService = mock(EntityService.class); - - for (int i = 0; i < ENTITY_TYPES.size(); i++) { - Mockito.when( - mockService.getEntityV2( - any(OperationContext.class), - any(), - Mockito.eq(ENTITY_URNS.get(i)), - Mockito.eq(Collections.singleton(CONTAINER_ASPECT_NAME)))) - .thenReturn(null); - } - - return mockService; - } - - private SearchService initMockSearchService() { - final SearchService mockSearchService = mock(SearchService.class); - - for (int i = 0; i < ENTITY_TYPES.size(); i++) { - Mockito.when( - mockSearchService.scrollAcrossEntities( - Mockito.any(OperationContext.class), - Mockito.eq(ImmutableList.of(ENTITY_TYPES.get(i))), - Mockito.eq("*"), - any(Filter.class), - Mockito.eq(null), - Mockito.eq(null), - Mockito.eq("5m"), - Mockito.eq(5000))) - .thenReturn( - new ScrollResult() - .setNumEntities(1) - .setEntities( - new SearchEntityArray(new SearchEntity().setEntity(ENTITY_URNS.get(i))))); - } - - return mockSearchService; - } -} diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java deleted file mode 100644 index 08f3dba12bd2a..0000000000000 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java +++ /dev/null @@ -1,406 +0,0 @@ -package com.linkedin.metadata.boot.steps; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableList; -import com.linkedin.common.AuditStamp; -import com.linkedin.common.BrowsePaths; -import com.linkedin.common.urn.Urn; -import com.linkedin.common.urn.UrnUtils; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.StringArray; -import com.linkedin.entity.Aspect; -import com.linkedin.entity.EntityResponse; -import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.entity.EnvelopedAspectMap; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.ListResult; -import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.EntitySpec; -import com.linkedin.metadata.models.EntitySpecBuilder; -import com.linkedin.metadata.models.EventSpec; -import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.query.ExtraInfo; -import com.linkedin.metadata.query.ExtraInfoArray; -import com.linkedin.metadata.query.ListResultMetadata; -import com.linkedin.metadata.search.utils.BrowsePathUtils; -import com.linkedin.metadata.snapshot.Snapshot; -import com.linkedin.mxe.MetadataChangeProposal; -import io.datahubproject.metadata.context.OperationContext; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.jetbrains.annotations.NotNull; -import org.mockito.Mockito; -import org.testng.annotations.Test; - -public class UpgradeDefaultBrowsePathsStepTest { - - private static final String VERSION_1 = "1"; - private static final String UPGRADE_URN = - String.format( - "urn:li:%s:%s", - Constants.DATA_HUB_UPGRADE_ENTITY_NAME, "upgrade-default-browse-paths-step"); - - @Test - public void testExecuteNoExistingBrowsePaths() throws Exception { - - final EntityService mockService = Mockito.mock(EntityService.class); - final EntityRegistry registry = new TestEntityRegistry(); - final OperationContext mockContext = mock(OperationContext.class); - when(mockContext.getEntityRegistry()).thenReturn(registry); - - final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); - Mockito.when( - mockService.getEntityV2( - any(OperationContext.class), - Mockito.eq(Constants.DATA_HUB_UPGRADE_ENTITY_NAME), - Mockito.eq(upgradeEntityUrn), - Mockito.eq(Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)))) - .thenReturn(null); - - final List browsePaths1 = Collections.emptyList(); - - Mockito.when( - mockService.listLatestAspects( - any(OperationContext.class), - Mockito.eq(Constants.DATASET_ENTITY_NAME), - Mockito.eq(Constants.BROWSE_PATHS_ASPECT_NAME), - Mockito.eq(0), - Mockito.eq(5000))) - .thenReturn( - new ListResult<>( - browsePaths1, - new ListResultMetadata().setExtraInfos(new ExtraInfoArray(Collections.emptyList())), - 0, - false, - 0, - 0, - 2)); - initMockServiceOtherEntities(mockService); - - UpgradeDefaultBrowsePathsStep upgradeDefaultBrowsePathsStep = - new UpgradeDefaultBrowsePathsStep(mockService); - upgradeDefaultBrowsePathsStep.execute(mockContext); - - Mockito.verify(mockService, Mockito.times(1)) - .listLatestAspects( - any(OperationContext.class), - Mockito.eq(Constants.DATASET_ENTITY_NAME), - Mockito.eq(Constants.BROWSE_PATHS_ASPECT_NAME), - Mockito.eq(0), - Mockito.eq(5000)); - // Verify that 4 aspects are ingested, 2 for the upgrade request / result, but none for - // ingesting - Mockito.verify(mockService, Mockito.times(2)) - .ingestProposal( - any(OperationContext.class), - any(MetadataChangeProposal.class), - any(), - Mockito.eq(false)); - } - - @Test - public void testExecuteFirstTime() throws Exception { - - Urn testUrn1 = - UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset1,PROD)"); - Urn testUrn2 = - UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset2,PROD)"); - - final EntityService mockService = Mockito.mock(EntityService.class); - final EntityRegistry registry = new TestEntityRegistry(); - final OperationContext mockContext = mock(OperationContext.class); - when(mockContext.getEntityRegistry()).thenReturn(registry); - - final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); - Mockito.when( - mockService.getEntityV2( - any(OperationContext.class), - Mockito.eq(Constants.DATA_HUB_UPGRADE_ENTITY_NAME), - Mockito.eq(upgradeEntityUrn), - Mockito.eq(Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)))) - .thenReturn(null); - final List browsePaths1 = - ImmutableList.of( - new BrowsePaths() - .setPaths( - new StringArray( - ImmutableList.of( - BrowsePathUtils.getLegacyDefaultBrowsePath(testUrn1, registry)))), - new BrowsePaths() - .setPaths( - new StringArray( - ImmutableList.of( - BrowsePathUtils.getLegacyDefaultBrowsePath(testUrn2, registry))))); - - final List extraInfos1 = - ImmutableList.of( - new ExtraInfo() - .setUrn(testUrn1) - .setVersion(0L) - .setAudit( - new AuditStamp().setActor(UrnUtils.getUrn("urn:li:corpuser:test")).setTime(0L)), - new ExtraInfo() - .setUrn(testUrn2) - .setVersion(0L) - .setAudit( - new AuditStamp() - .setActor(UrnUtils.getUrn("urn:li:corpuser:test")) - .setTime(0L))); - - Mockito.when( - mockService.listLatestAspects( - any(OperationContext.class), - Mockito.eq(Constants.DATASET_ENTITY_NAME), - Mockito.eq(Constants.BROWSE_PATHS_ASPECT_NAME), - Mockito.eq(0), - Mockito.eq(5000))) - .thenReturn( - new ListResult<>( - browsePaths1, - new ListResultMetadata().setExtraInfos(new ExtraInfoArray(extraInfos1)), - 2, - false, - 2, - 2, - 2)); - initMockServiceOtherEntities(mockService); - - UpgradeDefaultBrowsePathsStep upgradeDefaultBrowsePathsStep = - new UpgradeDefaultBrowsePathsStep(mockService); - upgradeDefaultBrowsePathsStep.execute(mockContext); - - Mockito.verify(mockService, Mockito.times(1)) - .listLatestAspects( - any(OperationContext.class), - Mockito.eq(Constants.DATASET_ENTITY_NAME), - Mockito.eq(Constants.BROWSE_PATHS_ASPECT_NAME), - Mockito.eq(0), - Mockito.eq(5000)); - // Verify that 4 aspects are ingested, 2 for the upgrade request / result and 2 for the browse - // pahts - Mockito.verify(mockService, Mockito.times(4)) - .ingestProposal( - any(OperationContext.class), - any(MetadataChangeProposal.class), - any(), - Mockito.eq(false)); - } - - @Test - public void testDoesNotRunWhenBrowsePathIsNotQualified() throws Exception { - // Test for browse paths that are not ingested - Urn testUrn3 = - UrnUtils.getUrn( - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset3,PROD)"); // Do not - // migrate - Urn testUrn4 = - UrnUtils.getUrn( - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset4,PROD)"); // Do not - // migrate - - final EntityService mockService = Mockito.mock(EntityService.class); - final EntityRegistry registry = new TestEntityRegistry(); - final OperationContext mockContext = mock(OperationContext.class); - when(mockContext.getEntityRegistry()).thenReturn(registry); - - final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); - Mockito.when( - mockService.getEntityV2( - any(OperationContext.class), - Mockito.eq(Constants.DATA_HUB_UPGRADE_ENTITY_NAME), - Mockito.eq(upgradeEntityUrn), - Mockito.eq(Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)))) - .thenReturn(null); - - final List browsePaths2 = - ImmutableList.of( - new BrowsePaths() - .setPaths( - new StringArray( - ImmutableList.of( - BrowsePathUtils.getDefaultBrowsePath(testUrn3, registry, '.')))), - new BrowsePaths() - .setPaths( - new StringArray( - ImmutableList.of( - BrowsePathUtils.getLegacyDefaultBrowsePath(testUrn4, registry), - BrowsePathUtils.getDefaultBrowsePath(testUrn4, registry, '.'))))); - - final List extraInfos2 = - ImmutableList.of( - new ExtraInfo() - .setUrn(testUrn3) - .setVersion(0L) - .setAudit( - new AuditStamp().setActor(UrnUtils.getUrn("urn:li:corpuser:test")).setTime(0L)), - new ExtraInfo() - .setUrn(testUrn4) - .setVersion(0L) - .setAudit( - new AuditStamp() - .setActor(UrnUtils.getUrn("urn:li:corpuser:test")) - .setTime(0L))); - - Mockito.when( - mockService.listLatestAspects( - any(OperationContext.class), - Mockito.eq(Constants.DATASET_ENTITY_NAME), - Mockito.eq(Constants.BROWSE_PATHS_ASPECT_NAME), - Mockito.eq(0), - Mockito.eq(5000))) - .thenReturn( - new ListResult<>( - browsePaths2, - new ListResultMetadata().setExtraInfos(new ExtraInfoArray(extraInfos2)), - 2, - false, - 2, - 2, - 2)); - initMockServiceOtherEntities(mockService); - - UpgradeDefaultBrowsePathsStep upgradeDefaultBrowsePathsStep = - new UpgradeDefaultBrowsePathsStep(mockService); - upgradeDefaultBrowsePathsStep.execute(mockContext); - - Mockito.verify(mockService, Mockito.times(1)) - .listLatestAspects( - any(OperationContext.class), - Mockito.eq(Constants.DATASET_ENTITY_NAME), - Mockito.eq(Constants.BROWSE_PATHS_ASPECT_NAME), - Mockito.eq(0), - Mockito.eq(5000)); - // Verify that 2 aspects are ingested, only those for the upgrade step - Mockito.verify(mockService, Mockito.times(2)) - .ingestProposal( - any(OperationContext.class), - any(MetadataChangeProposal.class), - any(), - Mockito.eq(false)); - } - - @Test - public void testDoesNotRunWhenAlreadyExecuted() throws Exception { - final EntityService mockService = Mockito.mock(EntityService.class); - final OperationContext mockContext = mock(OperationContext.class); - when(mockContext.getEntityRegistry()).thenReturn(mock(EntityRegistry.class)); - - final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); - com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest = - new com.linkedin.upgrade.DataHubUpgradeRequest().setVersion(VERSION_1); - Map upgradeRequestAspects = new HashMap<>(); - upgradeRequestAspects.put( - Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME, - new EnvelopedAspect().setValue(new Aspect(upgradeRequest.data()))); - EntityResponse response = - new EntityResponse().setAspects(new EnvelopedAspectMap(upgradeRequestAspects)); - Mockito.when( - mockService.getEntityV2( - any(OperationContext.class), - Mockito.eq(Constants.DATA_HUB_UPGRADE_ENTITY_NAME), - Mockito.eq(upgradeEntityUrn), - Mockito.eq(Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)))) - .thenReturn(response); - - UpgradeDefaultBrowsePathsStep step = new UpgradeDefaultBrowsePathsStep(mockService); - step.execute(mockContext); - - Mockito.verify(mockService, Mockito.times(0)) - .ingestProposal( - any(OperationContext.class), - any(MetadataChangeProposal.class), - any(AuditStamp.class), - Mockito.anyBoolean()); - } - - private void initMockServiceOtherEntities(EntityService mockService) { - List skippedEntityTypes = - ImmutableList.of( - Constants.DASHBOARD_ENTITY_NAME, - Constants.CHART_ENTITY_NAME, - Constants.DATA_FLOW_ENTITY_NAME, - Constants.DATA_JOB_ENTITY_NAME); - for (String entityType : skippedEntityTypes) { - Mockito.when( - mockService.listLatestAspects( - any(OperationContext.class), - Mockito.eq(entityType), - Mockito.eq(Constants.BROWSE_PATHS_ASPECT_NAME), - Mockito.eq(0), - Mockito.eq(5000))) - .thenReturn( - new ListResult<>( - Collections.emptyList(), - new ListResultMetadata() - .setExtraInfos(new ExtraInfoArray(Collections.emptyList())), - 0, - false, - 0, - 0, - 0)); - } - } - - public static class TestEntityRegistry implements EntityRegistry { - - private final Map entityNameToSpec; - - public TestEntityRegistry() { - entityNameToSpec = - new EntitySpecBuilder(EntitySpecBuilder.AnnotationExtractionMode.IGNORE_ASPECT_FIELDS) - .buildEntitySpecs(new Snapshot().schema()).stream() - .collect(Collectors.toMap(spec -> spec.getName().toLowerCase(), spec -> spec)); - } - - @Nonnull - @Override - public EntitySpec getEntitySpec(@Nonnull final String entityName) { - String lowercaseEntityName = entityName.toLowerCase(); - if (!entityNameToSpec.containsKey(lowercaseEntityName)) { - throw new IllegalArgumentException( - String.format("Failed to find entity with name %s in EntityRegistry", entityName)); - } - return entityNameToSpec.get(lowercaseEntityName); - } - - @Nullable - @Override - public EventSpec getEventSpec(@Nonnull String eventName) { - return null; - } - - @Nonnull - @Override - public Map getEntitySpecs() { - return entityNameToSpec; - } - - @NotNull - @Override - public Map getAspectSpecs() { - return new HashMap<>(); - } - - @Nonnull - @Override - public Map getEventSpecs() { - return Collections.emptyMap(); - } - - @NotNull - @Override - public AspectTemplateEngine getAspectTemplateEngine() { - return new AspectTemplateEngine(); - } - } -} From 027ae65d66038233e68e7fa2ff8ef369d1e6e516 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Sat, 28 Sep 2024 13:23:20 -0500 Subject: [PATCH 2/5] enable stacktrace --- .github/workflows/build-and-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 52148ef1b91f9..e9a3977e8c7f4 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -81,7 +81,7 @@ jobs: - name: Gradle build (and test) for NOT metadata ingestion if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} run: | - ./gradlew build \ + ./gradlew build --stacktrace --parallel \ -x :metadata-ingestion:build \ -x :metadata-ingestion:check \ -x :docs-website:build \ @@ -96,8 +96,7 @@ jobs: -x :metadata-ingestion-modules:gx-plugin:build \ -x :metadata-ingestion-modules:gx-plugin:check \ -x :datahub-frontend:build \ - -x :datahub-web-react:build \ - --parallel + -x :datahub-web-react:build - name: Gradle build (and test) for frontend if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }} run: | From 9dd8007317a05b29648644ab2d16cc98e1b9c3a3 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Sat, 28 Sep 2024 13:40:18 -0500 Subject: [PATCH 3/5] rerun-tasks --- .github/workflows/build-and-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e9a3977e8c7f4..e853041ab684f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -81,7 +81,7 @@ jobs: - name: Gradle build (and test) for NOT metadata ingestion if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} run: | - ./gradlew build --stacktrace --parallel \ + ./gradlew build \ -x :metadata-ingestion:build \ -x :metadata-ingestion:check \ -x :docs-website:build \ @@ -96,7 +96,8 @@ jobs: -x :metadata-ingestion-modules:gx-plugin:build \ -x :metadata-ingestion-modules:gx-plugin:check \ -x :datahub-frontend:build \ - -x :datahub-web-react:build + -x :datahub-web-react:build \ + --parallel --rerun-tasks - name: Gradle build (and test) for frontend if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }} run: | From 1fe77c6aa121144e56ae0ff8f456cf07e7292244 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Mon, 30 Sep 2024 10:38:19 -0500 Subject: [PATCH 4/5] test bypass build cache --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e853041ab684f..92e308274986a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -97,7 +97,7 @@ jobs: -x :metadata-ingestion-modules:gx-plugin:check \ -x :datahub-frontend:build \ -x :datahub-web-react:build \ - --parallel --rerun-tasks + --parallel --no-build-cache - name: Gradle build (and test) for frontend if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }} run: | From 85f06999a0895b72cff22dd6f318210f0ac61f23 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Mon, 30 Sep 2024 11:09:18 -0500 Subject: [PATCH 5/5] restore gradle job set test assertion --- .github/workflows/build-and-test.yml | 2 +- .../metadata/boot/steps/IngestEntityTypesStepTest.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 92e308274986a..52148ef1b91f9 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -97,7 +97,7 @@ jobs: -x :metadata-ingestion-modules:gx-plugin:check \ -x :datahub-frontend:build \ -x :datahub-web-react:build \ - --parallel --no-build-cache + --parallel - name: Gradle build (and test) for frontend if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }} run: | diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStepTest.java index a93bd8073a553..ab76f171aa7b1 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStepTest.java @@ -17,10 +17,18 @@ import io.datahubproject.test.metadata.context.TestOperationContexts; import org.jetbrains.annotations.NotNull; import org.mockito.Mockito; +import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; public class IngestEntityTypesStepTest { + @BeforeTest + public static void beforeTest() { + PathSpecBasedSchemaAnnotationVisitor.class + .getClassLoader() + .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false); + } + @Test public void testExecuteTestEntityRegistry() throws Exception { EntityRegistry testEntityRegistry = getTestEntityRegistry();