diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index 8106dfe72eaf..41558f7845a7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -57,6 +57,7 @@ public class SearchIndexApp extends AbstractNativeApplication { "dashboard", "topic", "pipeline", + "ingestionPipeline", "searchIndex", "user", "team", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java index 559ddc233841..c2a58b3a9371 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java @@ -350,7 +350,7 @@ public boolean matchIngestionPipelineState(List pipelineState) { for (FieldChange fieldChange : changeEvent.getChangeDescription().getFieldsUpdated()) { if (fieldChange.getName().equals("pipelineStatus") && fieldChange.getNewValue() != null) { PipelineStatus pipelineStatus = - JsonUtils.convertValue(fieldChange.getNewValue(), PipelineStatus.class); + JsonUtils.readOrConvertValue(fieldChange.getNewValue(), PipelineStatus.class); PipelineStatusType status = pipelineStatus.getPipelineState(); for (String givenStatus : pipelineState) { if (givenStatus.equalsIgnoreCase(status.value())) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/entity/IngestionPipelineFormatter.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/entity/IngestionPipelineFormatter.java index 214d7a12f272..e451908c5a6b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/entity/IngestionPipelineFormatter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/entity/IngestionPipelineFormatter.java @@ -22,6 +22,7 @@ import org.openmetadata.schema.type.FieldChange; import org.openmetadata.service.formatter.decorators.MessageDecorator; import org.openmetadata.service.formatter.util.FormatterUtil; +import org.openmetadata.service.util.JsonUtils; public class IngestionPipelineFormatter implements EntityFormatter { private static final String PIPELINE_STATUS_FIELD = "pipelineStatus"; @@ -41,10 +42,12 @@ public String format( private String transformIngestionPipelineStatus( MessageDecorator messageFormatter, FieldChange fieldChange, EntityInterface entity) { String ingestionPipelineName = entity.getName(); - PipelineStatus status = (PipelineStatus) fieldChange.getNewValue(); + PipelineStatus status = + JsonUtils.readOrConvertValue(fieldChange.getNewValue(), PipelineStatus.class); if (status != null) { + // In case of running String date = - new SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(new Date(status.getEndDate())); + new SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(new Date(status.getTimestamp())); String format = String.format( "Ingestion Pipeline %s %s at %s", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java index f2bc5f5982a1..eeb568659d60 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java @@ -614,7 +614,7 @@ private List getSubscriptions( List result = new ArrayList<>(); subscriptions.forEach( subscription -> { - if (subscription.getId() == null) { + if (nullOrEmpty(subscription.getId())) { subscription.withId(UUID.randomUUID()); } result.add(subscription); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java index a0426b0e5f05..fddf74288b7c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java @@ -28,6 +28,7 @@ import org.openmetadata.schema.entity.services.PipelineService; import org.openmetadata.schema.entity.services.SearchService; import org.openmetadata.schema.entity.services.StorageService; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.tests.TestCase; @@ -49,6 +50,7 @@ import org.openmetadata.service.search.indexes.EntityReportDataIndex; import org.openmetadata.service.search.indexes.GlossaryIndex; import org.openmetadata.service.search.indexes.GlossaryTermIndex; +import org.openmetadata.service.search.indexes.IngestionPipelineIndex; import org.openmetadata.service.search.indexes.MessagingServiceIndex; import org.openmetadata.service.search.indexes.MetadataServiceIndex; import org.openmetadata.service.search.indexes.MlModelIndex; @@ -82,6 +84,7 @@ public SearchIndex buildIndex(String entityType, Object entity) { case Entity.DASHBOARD -> new DashboardIndex((Dashboard) entity); case Entity.TOPIC -> new TopicIndex((Topic) entity); case Entity.PIPELINE -> new PipelineIndex((Pipeline) entity); + case Entity.INGESTION_PIPELINE -> new IngestionPipelineIndex((IngestionPipeline) entity); case Entity.USER -> new UserIndex((User) entity); case Entity.TEAM -> new TeamIndex((Team) entity); case Entity.GLOSSARY -> new GlossaryIndex((Glossary) entity); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java new file mode 100644 index 000000000000..2432cfe82714 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java @@ -0,0 +1,72 @@ +package org.openmetadata.service.search.indexes; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.service.Entity; +import org.openmetadata.service.search.ParseTags; +import org.openmetadata.service.search.SearchIndexUtils; +import org.openmetadata.service.search.models.SearchSuggest; +import org.openmetadata.service.util.JsonUtils; + +public class IngestionPipelineIndex implements SearchIndex { + final IngestionPipeline ingestionPipeline; + final List excludeFields = + List.of("changeDescription", "sourceConfig", "openMetadataServerConnection", "airflowConfig"); + + public IngestionPipelineIndex(IngestionPipeline ingestionPipeline) { + this.ingestionPipeline = ingestionPipeline; + } + + public Map buildESDoc() { + Map doc = JsonUtils.getMap(ingestionPipeline); + SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + List suggest = new ArrayList<>(); + List serviceSuggest = new ArrayList<>(); + suggest.add( + SearchSuggest.builder().input(ingestionPipeline.getFullyQualifiedName()).weight(5).build()); + suggest.add( + SearchSuggest.builder().input(ingestionPipeline.getDisplayName()).weight(10).build()); + serviceSuggest.add( + SearchSuggest.builder().input(ingestionPipeline.getService().getName()).weight(5).build()); + ParseTags parseTags = + new ParseTags(Entity.getEntityTags(Entity.INGESTION_PIPELINE, ingestionPipeline)); + doc.put( + "name", + ingestionPipeline.getName() != null + ? ingestionPipeline.getName() + : ingestionPipeline.getDisplayName()); + doc.put( + "displayName", + ingestionPipeline.getDisplayName() != null + ? ingestionPipeline.getDisplayName() + : ingestionPipeline.getName()); + doc.put("followers", SearchIndexUtils.parseFollowers(ingestionPipeline.getFollowers())); + doc.put("tags", parseTags.getTags()); + doc.put("tier", parseTags.getTierTag()); + doc.put("suggest", suggest); + doc.put("service_suggest", serviceSuggest); + doc.put("entityType", Entity.INGESTION_PIPELINE); + doc.put( + "totalVotes", + CommonUtil.nullOrEmpty(ingestionPipeline.getVotes()) + ? 0 + : ingestionPipeline.getVotes().getUpVotes() + - ingestionPipeline.getVotes().getDownVotes()); + doc.put( + "fqnParts", + getFQNParts( + ingestionPipeline.getFullyQualifiedName(), + suggest.stream().map(SearchSuggest::getInput).toList())); + doc.put("owner", getEntityWithDisplayName(ingestionPipeline.getOwner())); + doc.put("service", getEntityWithDisplayName(ingestionPipeline.getService())); + doc.put("domain", getEntityWithDisplayName(ingestionPipeline.getDomain())); + return doc; + } + + public static Map getFields() { + return SearchIndex.getDefaultFields(); + } +} diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/ingestion_pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/ingestion_pipeline_index_mapping.json new file mode 100644 index 000000000000..d535f835dbd6 --- /dev/null +++ b/openmetadata-service/src/main/resources/elasticsearch/en/ingestion_pipeline_index_mapping.json @@ -0,0 +1,313 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, + "om_ngram": { + "tokenizer": "ngram", + "min_gram": 3, + "max_gram": 10, + "filter": [ + "lowercase" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fqnParts": { + "type": "keyword" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "index_options": "docs" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "domain" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "owner": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "deleted": { + "type": "text" + }, + "deployed": { + "type": "text" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "pipelineType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "entityType": { + "type": "keyword" + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "service_suggest": { + "type": "completion" + }, + "totalVotes": { + "type": "long" + }, + "votes" : { + "type": "object" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json index 6247826dcf6f..e6ccbd373b8c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json @@ -220,5 +220,11 @@ "indexMappingFile": "/elasticsearch/%s/test_case_resolution_status_index_mapping.json", "alias": "testCaseResolutionStatus", "parentAliases": [] + }, + "ingestionPipeline": { + "indexName": "ingestion_pipeline_search_index", + "indexMappingFile": "/elasticsearch/%s/ingestion_pipeline_index_mapping.json", + "alias": "ingestionPipeline", + "parentAliases": ["all", "dataAsset"] } } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json new file mode 100644 index 000000000000..42d851f2f296 --- /dev/null +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json @@ -0,0 +1,288 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer_jp" : { + "tokenizer" : "kuromoji_tokenizer", + "type" : "custom", + "filter" : [ + "kuromoji_baseform", + "kuromoji_part_of_speech", + "kuromoji_number", + "kuromoji_stemmer" + ] + }, + "om_ngram": { + "tokenizer": "ngram", + "min_gram": 1, + "max_gram": 2, + "filter": [ + "lowercase" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fqnParts": { + "type": "keyword" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer_jp" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "domain" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "owner": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "deleted": { + "type": "text" + }, + "deployed": { + "type": "text" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "pipelineType": { + "type": "keyword" + }, + "entityType": { + "type": "keyword" + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "service_suggest": { + "type": "completion" + }, + "totalVotes": { + "type": "long" + }, + "votes" : { + "type": "object" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/ingestion_pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/ingestion_pipeline_index_mapping.json new file mode 100644 index 000000000000..8675925157f8 --- /dev/null +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/ingestion_pipeline_index_mapping.json @@ -0,0 +1,271 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fqnParts": { + "type": "keyword" + }, + "displayName": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart" + }, + "description": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "domain" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "owner": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "deleted": { + "type": "text" + }, + "deployed": { + "type": "text" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "pipelineType": { + "type": "keyword" + }, + "entityType": { + "type": "keyword" + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "service_suggest": { + "type": "completion" + }, + "totalVotes": { + "type": "long" + }, + "votes" : { + "type": "object" + } + } + } + +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json index 328d8aab64bf..bf6922a0c327 100644 --- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json @@ -7,6 +7,7 @@ "dashboard", "topic", "pipeline", + "ingestionPipeline", "searchIndex", "user", "team", diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json index 07098b7e848e..841d53e5c453 100644 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json @@ -21,6 +21,7 @@ "dashboard", "topic", "pipeline", + "ingestionPipeline", "searchIndex", "user", "team", diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.js index f9cd2c03dcb5..43c97b9ecbc0 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.js @@ -28,6 +28,8 @@ export const TEST_CASE_NAME = `0%test-case-${uuid()}`; export const TABLE_FQN = `${DATABASE_SERVICE.entity.databaseSchema}.${DATABASE_SERVICE.entity.name}`; export const TEST_SUITE_FQN = `${TABLE_FQN}.testSuite`; +export const INGESTION_PIPELINE_NAME = `0-cy-ingestion-pipeline-${uuid()}`; + export const OBSERVABILITY_CREATION_DETAILS = { table: { triggerDisplayName: 'Table', @@ -74,6 +76,53 @@ export const OBSERVABILITY_CREATION_DETAILS = { }, ], }, + ingestionPipeline: { + triggerDisplayName: 'Ingestion Pipeline', + filters: [ + { + name: 'Ingestion Pipeline Name', + inputSelector: 'fqn-list-select', + inputValue: INGESTION_PIPELINE_NAME, + exclude: false, + }, + { + name: 'Domain', + inputSelector: 'domain-select', + inputValue: DOMAIN_CREATION_DETAILS.name, + exclude: false, + }, + { + name: 'Owner Name', + inputSelector: 'owner-name-select', + inputValue: `${USER_DETAILS.firstName}${USER_DETAILS.lastName}`, + exclude: true, + }, + ], + actions: [ + { + name: 'Get Ingestion Pipeline Status Updates', + inputs: [ + { + inputSelector: 'pipeline-status-select', + inputValue: 'Queued', + }, + ], + exclude: false, + }, + ], + destinations: [ + { + mode: 'internal', + category: 'Owners', + type: 'Email', + }, + { + mode: 'external', + category: 'Email', + inputValue: 'test@example.com', + }, + ], + }, testCase: { triggerDisplayName: 'Test Case', filters: [ diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.js index 0b99babc783e..553c30e3eabf 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.js @@ -31,15 +31,12 @@ import { toastNotification, verifyResponseStatusCode, } from '../../common/common'; -import { - createEntityTable, - createSingleLevelEntity, - hardDeleteService, -} from '../../common/EntityUtils'; +import { createEntityTable, hardDeleteService } from '../../common/EntityUtils'; import { ALERT_DESCRIPTION, ALERT_NAME, ALERT_UPDATED_DESCRIPTION, + INGESTION_PIPELINE_NAME, OBSERVABILITY_CREATION_DETAILS, TABLE_FQN, TEST_CASE_NAME, @@ -108,9 +105,42 @@ describe('Observability Alert Flow', () => { }); // Create a pipeline - createSingleLevelEntity({ - token, - ...PIPELINE_SERVICE, + cy.request({ + method: 'POST', + url: `/api/v1/services/${PIPELINE_SERVICE.serviceType}`, + headers: { Authorization: `Bearer ${token}` }, + body: PIPELINE_SERVICE.service, + }).then((pipelineServiceResponse) => { + data.pipelineService = pipelineServiceResponse.body; + + cy.request({ + method: 'POST', + url: `/api/v1/${PIPELINE_SERVICE.entityType}`, + headers: { Authorization: `Bearer ${token}` }, + body: PIPELINE_SERVICE.entity, + }); + + // Create a ingestion pipeline + cy.request({ + method: 'POST', + url: `/api/v1/services/ingestionPipelines`, + headers: { Authorization: `Bearer ${token}` }, + body: { + airflowConfig: {}, + loggerLevel: 'INFO', + name: INGESTION_PIPELINE_NAME, + pipelineType: 'metadata', + service: { + id: data.pipelineService.id, + type: 'pipelineService', + }, + sourceConfig: { + config: {}, + }, + }, + }).then((ingestionPipelineResponse) => { + data.ingestionPipeline = ingestionPipelineResponse.body; + }); }); // Create a new user @@ -154,6 +184,13 @@ describe('Observability Alert Flow', () => { headers: { Authorization: `Bearer ${token}` }, }); + // Delete ingestion pipeline + cy.request({ + method: 'DELETE', + url: `/api/v1/services/ingestionPipelines/${data.ingestionPipeline.id}?hardDelete=true`, + headers: { Authorization: `Bearer ${token}` }, + }); + // Delete created services hardDeleteService({ token, diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts index c40fa61a41e1..ad3de78ed43a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts @@ -61,6 +61,7 @@ export enum EntityType { GOVERN = 'govern', ALL = 'all', CUSTOM_METRIC = 'customMetric', + INGESTION_PIPELINE = 'ingestionPipeline', } export enum AssetsType { diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/search.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/search.enum.ts index 1325b956a7ce..0a6333fc2134 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/search.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/search.enum.ts @@ -41,4 +41,5 @@ export enum SearchIndex { STORED_PROCEDURE = 'stored_procedure_search_index', DASHBOARD_DATA_MODEL = 'dashboard_data_model_search_index', DATA_PRODUCT = 'data_product_search_index', + INGESTION_PIPELINE = 'ingestion_pipeline_search_index', } diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts index 18774e8fd6e6..aba89f2435fd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts @@ -32,6 +32,7 @@ import { DataProduct } from '../generated/entity/domains/dataProduct'; import { Domain } from '../generated/entity/domains/domain'; import { DashboardService } from '../generated/entity/services/dashboardService'; import { DatabaseService } from '../generated/entity/services/databaseService'; +import { IngestionPipeline } from '../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { MessagingService } from '../generated/entity/services/messagingService'; import { MlmodelService } from '../generated/entity/services/mlmodelService'; import { PipelineService } from '../generated/entity/services/pipelineService'; @@ -122,6 +123,10 @@ export interface TestCaseSearchSource } // extends EntityInterface export interface TestSuiteSearchSource extends SearchSourceBase, TestSuite {} +export interface IngestionPipelineSearchSource + extends SearchSourceBase, + IngestionPipeline {} + export interface DatabaseServiceSearchSource extends SearchSourceBase, DatabaseService {} @@ -203,6 +208,7 @@ export type SearchIndexSearchSourceMapping = { [SearchIndex.DASHBOARD_DATA_MODEL]: DashboardDataModelSearchSource; [SearchIndex.DATA_PRODUCT]: DataProductSearchSource; [SearchIndex.TEST_SUITE]: TestSuiteSearchSource; + [SearchIndex.INGESTION_PIPELINE]: IngestionPipelineSearchSource; }; export type SearchRequest< diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json index b2b79db2eb23..4ce5bc84bd2f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json @@ -21,6 +21,7 @@ "dashboard", "topic", "pipeline", + "ingestionPipeline", "searchIndex", "user", "team", diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts index e7e7f95a8655..eba3a9d5a0c6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts @@ -96,6 +96,7 @@ class SearchClassBase { [EntityType.TEST_CASE]: SearchIndex.TEST_CASE, [EntityType.TEST_SUITE]: SearchIndex.TEST_SUITE, [EntityType.GLOSSARY]: SearchIndex.GLOSSARY, + [EntityType.INGESTION_PIPELINE]: SearchIndex.INGESTION_PIPELINE, }; }