fix(analytics): index description so analytics are correct (#11224)

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDomainDescriptionConfig.java

@@ -0,0 +1,29 @@
+package com.linkedin.datahub.upgrade.config;
+
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.datahub.upgrade.system.domaindescription.ReindexDomainDescription;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class)
+public class ReindexDomainDescriptionConfig {
+
+ @Bean
+ public NonBlockingSystemUpgrade reindexDomainDescription(
+ final OperationContext opContext,
+ final EntityService<?> entityService,
+ final AspectDao aspectDao,
+ @Value("${systemUpdate.domainDescription.enabled}") final boolean enabled,
+ @Value("${systemUpdate.domainDescription.batchSize}") final Integer batchSize,
+ @Value("${systemUpdate.domainDescription.delayMs}") final Integer delayMs,
+ @Value("${systemUpdate.domainDescription.limit}") final Integer limit) {
+ return new ReindexDomainDescription(
+ opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit);
+ }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescription.java

@@ -0,0 +1,49 @@
+package com.linkedin.datahub.upgrade.system.domaindescription;
+
+import com.google.common.collect.ImmutableList;
+import com.linkedin.datahub.upgrade.UpgradeStep;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * A job that reindexes all domain aspects as part of reindexing descriptions This is required to
+ * fix the analytics for domains
+ */
+@Slf4j
+public class ReindexDomainDescription implements NonBlockingSystemUpgrade {
+
+ private final List<UpgradeStep> _steps;
+
+ public ReindexDomainDescription(
+ @Nonnull OperationContext opContext,
+ EntityService<?> entityService,
+ AspectDao aspectDao,
+ boolean enabled,
+ Integer batchSize,
+ Integer batchDelayMs,
+ Integer limit) {
+ if (enabled) {
+ _steps =
+ ImmutableList.of(
+ new ReindexDomainDescriptionStep(
+ opContext, entityService, aspectDao, batchSize, batchDelayMs, limit));
+ } else {
+ _steps = ImmutableList.of();
+ }
+ }
+
+ @Override
+ public String id() {
+ return this.getClass().getName();
+ }
+
+ @Override
+ public List<UpgradeStep> steps() {
+ return _steps;
+ }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescriptionStep.java

@@ -0,0 +1,42 @@
+package com.linkedin.datahub.upgrade.system.domaindescription;
+
+import static com.linkedin.metadata.Constants.*;
+
+import com.linkedin.datahub.upgrade.system.AbstractMCLStep;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.Nullable;
+
+@Slf4j
+public class ReindexDomainDescriptionStep extends AbstractMCLStep {
+
+ public ReindexDomainDescriptionStep(
+ OperationContext opContext,
+ EntityService<?> entityService,
+ AspectDao aspectDao,
+ Integer batchSize,
+ Integer batchDelayMs,
+ Integer limit) {
+ super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit);
+ }
+
+ @Override
+ public String id() {
+ return "domain-description-v1";
+ }
+
+ @Nonnull
+ @Override
+ protected String getAspectName() {
+ return DOMAIN_PROPERTIES_ASPECT_NAME;
+ }
+
+ @Nullable
+ @Override
+ protected String getUrnLike() {
+ return "urn:li:" + DOMAIN_ENTITY_NAME + ":%";
+ }
+}

metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl

@@ -26,6 +26,10 @@ record DomainProperties includes CustomProperties {
  /**
  * Description of the Domain
  */
+ @Searchable = {
+ "fieldType": "TEXT",
+ "hasValuesFieldName": "hasDescription"
+ }
  description: optional string
 
  /**

metadata-service/configuration/src/main/resources/application.yaml

@@ -335,6 +335,11 @@ systemUpdate:
  batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_BATCH_SIZE:1000}
  delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_DELAY_MS:30000}
  limit: ${BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_LIMIT:0}
+ domainDescription:
+ enabled: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_ENABLED:true}
+ batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_BATCH_SIZE:1000}
+ delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_DELAY_MS:30000}
+ limit: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_CLL_LIMIT:0}
  browsePathsV2:
  enabled: ${BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_ENABLED:true}
  batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_BATCH_SIZE:5000}