Skip to content

Commit

Permalink
Merge branch 'main' into feat-#15380
Browse files Browse the repository at this point in the history
  • Loading branch information
Sachin-chaurasiya authored Dec 20, 2024
2 parents b32ffd9 + d084243 commit bab1ce4
Show file tree
Hide file tree
Showing 130 changed files with 2,052 additions and 1,148 deletions.
6 changes: 3 additions & 3 deletions ingestion/src/metadata/ingestion/api/topology_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -418,9 +418,9 @@ def yield_and_update_context(
if not entity:
# Safe access to Entity Request name
raise MissingExpectedEntityAckException(
f"Missing ack back from [{stage.type_.__name__}: {entity_fqn}] - "
"Possible causes are changes in the server Fernet key or mismatched JSON Schemas "
"for the service connection."
f"We are trying to create a [{stage.type_.__name__}] with FQN [{entity_fqn}],"
" but we got no Entity back from the API. Checking for errors in the OpenMetadata Sink could help"
" validate if the Entity was properly created or not."
)

self.context.get().update_context_name(stage=stage, right=right)
Expand Down
7 changes: 7 additions & 0 deletions ingestion/src/metadata/utils/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
from metadata.generated.schema.entity.data.metric import Metric
from metadata.generated.schema.entity.data.mlmodel import MlModel
from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.searchIndex import SearchIndex
Expand Down Expand Up @@ -153,6 +156,10 @@
# Domain
"domain": Domain,
"dataProduct": DataProduct,
# Governance
"metric": Metric,
"glossary": Glossary,
"glossaryTerm": GlossaryTerm,
}

ENTITY_REFERENCE_TYPE_MAP = {
Expand Down
4 changes: 3 additions & 1 deletion ingestion/src/metadata/utils/elasticsearch.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
from metadata.generated.schema.entity.data.metric import Metric
from metadata.generated.schema.entity.data.mlmodel import MlModel
from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.query import Query
Expand Down Expand Up @@ -62,12 +63,13 @@
Topic.__name__: "topic_search_index",
Pipeline.__name__: "pipeline_search_index",
Glossary.__name__: "glossary_search_index",
GlossaryTerm.__name__: "glossary_search_index",
GlossaryTerm.__name__: "glossary_term_search_index",
MlModel.__name__: "mlmodel_search_index",
Tag.__name__: "tag_search_index",
Container.__name__: "container_search_index",
Query.__name__: "query_search_index",
ReportData.__name__: "entity_report_data_index",
Metric.__name__: "metric_search_index",
"web_analytic_user_activity_report": "web_analytic_user_activity_report_data_index",
"web_analytic_entity_view_report": "web_analytic_entity_view_report_data_index",
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.governance.workflows.WorkflowHandler;
import org.openmetadata.service.jdbi3.FeedRepository;
import org.openmetadata.service.resources.feeds.FeedResource;
import org.openmetadata.service.resources.feeds.FeedMapper;
import org.openmetadata.service.resources.feeds.MessageParser;
import org.openmetadata.service.util.WebsocketNotificationHandler;

Expand Down Expand Up @@ -91,7 +91,7 @@ private Thread createApprovalTask(GlossaryTerm entity, List<EntityReference> ass
} catch (EntityNotFoundException ex) {
TaskDetails taskDetails =
new TaskDetails()
.withAssignees(FeedResource.formatAssignees(assignees))
.withAssignees(FeedMapper.formatAssignees(assignees))
.withType(TaskType.RequestApproval)
.withStatus(TaskStatus.Open);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2053,11 +2053,12 @@ public final void validateUsers(List<EntityReference> entityReferences) {
}
}

private boolean validateIfAllRefsAreEntityType(List<EntityReference> list, String entityType) {
private static boolean validateIfAllRefsAreEntityType(
List<EntityReference> list, String entityType) {
return list.stream().allMatch(obj -> obj.getType().equals(entityType));
}

public final void validateReviewers(List<EntityReference> entityReferences) {
public static void validateReviewers(List<EntityReference> entityReferences) {
if (!nullOrEmpty(entityReferences)) {
boolean areAllTeam = validateIfAllRefsAreEntityType(entityReferences, TEAM);
boolean areAllUsers = validateIfAllRefsAreEntityType(entityReferences, USER);
Expand Down Expand Up @@ -2415,7 +2416,7 @@ protected void checkSystemEntityDeletion(T entity) {
}
}

public final List<EntityReference> validateOwners(List<EntityReference> owners) {
public static List<EntityReference> validateOwners(List<EntityReference> owners) {
if (nullOrEmpty(owners)) {
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import org.openmetadata.schema.EntityTimeSeriesInterface;
import org.openmetadata.schema.api.feed.CloseTask;
import org.openmetadata.schema.api.feed.ResolveTask;
import org.openmetadata.schema.api.tests.CreateTestCaseResult;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.tests.TestCase;
Expand Down Expand Up @@ -304,26 +303,10 @@ private void updateLogicalTestSuite(UUID testSuiteId) {

public RestUtil.PutResponse<TestCaseResult> addTestCaseResult(
String updatedBy, UriInfo uriInfo, String fqn, TestCaseResult testCaseResult) {
// TODO: REMOVED ONCE DEPRECATED IN TEST CASE RESOURCE
CreateTestCaseResult createTestCaseResult =
new CreateTestCaseResult()
.withTimestamp(testCaseResult.getTimestamp())
.withTestCaseStatus(testCaseResult.getTestCaseStatus())
.withResult(testCaseResult.getResult())
.withSampleData(testCaseResult.getSampleData())
.withTestResultValue(testCaseResult.getTestResultValue())
.withPassedRows(testCaseResult.getPassedRows())
.withFailedRows(testCaseResult.getFailedRows())
.withPassedRowsPercentage(testCaseResult.getPassedRowsPercentage())
.withFailedRowsPercentage(testCaseResult.getFailedRowsPercentage())
.withIncidentId(testCaseResult.getIncidentId())
.withMaxBound(testCaseResult.getMaxBound())
.withMinBound(testCaseResult.getMinBound());

TestCaseResultRepository testCaseResultRepository =
(TestCaseResultRepository) Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT);
Response response =
testCaseResultRepository.addTestCaseResult(updatedBy, uriInfo, fqn, createTestCaseResult);
testCaseResultRepository.addTestCaseResult(updatedBy, uriInfo, fqn, testCaseResult);
return new RestUtil.PutResponse<>(
Response.Status.CREATED, (TestCaseResult) response.getEntity(), ENTITY_UPDATED);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import javax.ws.rs.core.UriInfo;
import lombok.SneakyThrows;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.api.tests.CreateTestCaseResult;
import org.openmetadata.schema.tests.ResultSummary;
import org.openmetadata.schema.tests.TestCase;
import org.openmetadata.schema.tests.TestSuite;
Expand Down Expand Up @@ -80,9 +79,8 @@ public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, L
}

public Response addTestCaseResult(
String updatedBy, UriInfo uriInfo, String fqn, CreateTestCaseResult createTestCaseResult) {
String updatedBy, UriInfo uriInfo, String fqn, TestCaseResult testCaseResult) {
TestCase testCase = Entity.getEntityByName(TEST_CASE, fqn, "", Include.ALL);
TestCaseResult testCaseResult = getTestCaseResult(createTestCaseResult, testCase);
if (testCaseResult.getTestCaseStatus() == TestCaseStatus.Success) {
testCaseRepository.deleteTestCaseFailedRowsSample(testCase.getId());
}
Expand Down Expand Up @@ -299,24 +297,4 @@ public boolean hasTestCaseFailure(String fqn) throws IOException {
.anyMatch(
testCaseResult -> testCaseResult.getTestCaseStatus().equals(TestCaseStatus.Failed));
}

private TestCaseResult getTestCaseResult(
CreateTestCaseResult createTestCaseResults, TestCase testCase) {
RestUtil.validateTimestampMilliseconds(createTestCaseResults.getTimestamp());
return new TestCaseResult()
.withId(UUID.randomUUID())
.withTestCaseFQN(testCase.getFullyQualifiedName())
.withTimestamp(createTestCaseResults.getTimestamp())
.withTestCaseStatus(createTestCaseResults.getTestCaseStatus())
.withResult(createTestCaseResults.getResult())
.withSampleData(createTestCaseResults.getSampleData())
.withTestResultValue(createTestCaseResults.getTestResultValue())
.withPassedRows(createTestCaseResults.getPassedRows())
.withFailedRows(createTestCaseResults.getFailedRows())
.withPassedRowsPercentage(createTestCaseResults.getPassedRowsPercentage())
.withFailedRowsPercentage(createTestCaseResults.getFailedRowsPercentage())
.withIncidentId(createTestCaseResults.getIncidentId())
.withMaxBound(createTestCaseResults.getMaxBound())
.withMinBound(createTestCaseResults.getMinBound());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package org.openmetadata.service.mapper;

import static org.openmetadata.schema.type.Include.NON_DELETED;
import static org.openmetadata.service.jdbi3.EntityRepository.validateOwners;
import static org.openmetadata.service.jdbi3.EntityRepository.validateReviewers;
import static org.openmetadata.service.util.EntityUtil.getEntityReferences;

import java.util.List;
import java.util.UUID;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.CreateEntity;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;

public interface EntityMapper<T extends EntityInterface, C extends CreateEntity> {
T createToEntity(C create, String user);

default T copy(T entity, CreateEntity request, String updatedBy) {
List<EntityReference> owners = validateOwners(request.getOwners());
EntityReference domain = validateDomain(request.getDomain());
validateReviewers(request.getReviewers());
entity.setId(UUID.randomUUID());
entity.setName(request.getName());
entity.setDisplayName(request.getDisplayName());
entity.setDescription(request.getDescription());
entity.setOwners(owners);
entity.setDomain(domain);
entity.setTags(request.getTags());
entity.setDataProducts(getEntityReferences(Entity.DATA_PRODUCT, request.getDataProducts()));
entity.setLifeCycle(request.getLifeCycle());
entity.setExtension(request.getExtension());
entity.setUpdatedBy(updatedBy);
entity.setUpdatedAt(System.currentTimeMillis());
entity.setReviewers(request.getReviewers());
return entity;
}

default EntityReference validateDomain(String domainFqn) {
if (CommonUtil.nullOrEmpty(domainFqn)) {
return null;
}
return Entity.getEntityReferenceByName(Entity.DOMAIN, domainFqn, NON_DELETED);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package org.openmetadata.service.mapper;

import org.openmetadata.schema.EntityTimeSeriesInterface;

public interface EntityTimeSeriesMapper<T extends EntityTimeSeriesInterface, C> {
T createToEntity(C create, String user);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package org.openmetadata.service.resources.analytics;

import org.openmetadata.schema.analytics.WebAnalyticEvent;
import org.openmetadata.schema.api.tests.CreateWebAnalyticEvent;
import org.openmetadata.service.mapper.EntityMapper;

public class WebAnalyticEventMapper
implements EntityMapper<WebAnalyticEvent, CreateWebAnalyticEvent> {
@Override
public WebAnalyticEvent createToEntity(CreateWebAnalyticEvent create, String user) {
return copy(new WebAnalyticEvent(), create, user)
.withName(create.getName())
.withDisplayName(create.getDisplayName())
.withDescription(create.getDescription())
.withEventType(create.getEventType());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public class WebAnalyticEventResource
public static final String COLLECTION_PATH = WebAnalyticEventRepository.COLLECTION_PATH;
static final String FIELDS = "owners";
private static final Pattern HTML_PATTERN = Pattern.compile(".*\\<[^>]+>.*", Pattern.DOTALL);
private final WebAnalyticEventMapper mapper = new WebAnalyticEventMapper();

public WebAnalyticEventResource(Authorizer authorizer, Limits limits) {
super(Entity.WEB_ANALYTIC_EVENT, authorizer, limits);
Expand Down Expand Up @@ -169,7 +170,7 @@ public Response create(
@Context SecurityContext securityContext,
@Valid CreateWebAnalyticEvent create) {
WebAnalyticEvent webAnalyticEvent =
getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName());
mapper.createToEntity(create, securityContext.getUserPrincipal().getName());
return create(uriInfo, securityContext, webAnalyticEvent);
}

Expand All @@ -192,7 +193,7 @@ public Response createOrUpdate(
@Context SecurityContext securityContext,
@Valid CreateWebAnalyticEvent create) {
WebAnalyticEvent webAnalyticEvent =
getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName());
mapper.createToEntity(create, securityContext.getUserPrincipal().getName());
return createOrUpdate(uriInfo, securityContext, webAnalyticEvent);
}

Expand Down Expand Up @@ -554,15 +555,6 @@ public ResultList<WebAnalyticEventData> listWebAnalyticEventData(
return repository.getWebAnalyticEventData(eventType, startTs, endTs);
}

private WebAnalyticEvent getWebAnalyticEvent(CreateWebAnalyticEvent create, String user) {
return repository
.copy(new WebAnalyticEvent(), create, user)
.withName(create.getName())
.withDisplayName(create.getDisplayName())
.withDescription(create.getDescription())
.withEventType(create.getEventType());
}

public static WebAnalyticEventData sanitizeWebAnalyticEventData(
WebAnalyticEventData webAnalyticEventDataInput) {
Object inputData = webAnalyticEventDataInput.getEventData();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.openmetadata.service.resources.apis;

import static org.openmetadata.service.util.EntityUtil.getEntityReference;
import static org.openmetadata.service.util.EntityUtil.getEntityReferences;

import org.openmetadata.schema.api.data.CreateAPICollection;
import org.openmetadata.schema.entity.data.APICollection;
import org.openmetadata.service.Entity;
import org.openmetadata.service.mapper.EntityMapper;

public class APICollectionMapper implements EntityMapper<APICollection, CreateAPICollection> {
@Override
public APICollection createToEntity(CreateAPICollection create, String user) {
return copy(new APICollection(), create, user)
.withService(getEntityReference(Entity.API_SERVICE, create.getService()))
.withEndpointURL(create.getEndpointURL())
.withApiEndpoints(getEntityReferences(Entity.API_ENDPOINT, create.getApiEndpoints()))
.withSourceHash(create.getSourceHash());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
@Collection(name = "apiCollections")
public class APICollectionResource extends EntityResource<APICollection, APICollectionRepository> {
public static final String COLLECTION_PATH = "v1/apiCollections/";
private final APICollectionMapper mapper = new APICollectionMapper();
static final String FIELDS = "owners,apiEndpoints,tags,extension,domain,sourceHash";

@Override
Expand Down Expand Up @@ -310,7 +311,7 @@ public Response create(
@Context SecurityContext securityContext,
@Valid CreateAPICollection create) {
APICollection apiCollection =
getAPICollection(create, securityContext.getUserPrincipal().getName());
mapper.createToEntity(create, securityContext.getUserPrincipal().getName());
return create(uriInfo, securityContext, apiCollection);
}

Expand Down Expand Up @@ -392,7 +393,7 @@ public Response createOrUpdate(
@Context SecurityContext securityContext,
@Valid CreateAPICollection create) {
APICollection apiCollection =
getAPICollection(create, securityContext.getUserPrincipal().getName());
mapper.createToEntity(create, securityContext.getUserPrincipal().getName());
return createOrUpdate(uriInfo, securityContext, apiCollection);
}

Expand Down Expand Up @@ -510,13 +511,4 @@ public Response restore(
@Valid RestoreEntity restore) {
return restoreEntity(uriInfo, securityContext, restore.getId());
}

private APICollection getAPICollection(CreateAPICollection create, String user) {
return repository
.copy(new APICollection(), create, user)
.withService(getEntityReference(Entity.API_SERVICE, create.getService()))
.withEndpointURL(create.getEndpointURL())
.withApiEndpoints(getEntityReferences(Entity.API_ENDPOINT, create.getApiEndpoints()))
.withSourceHash(create.getSourceHash());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package org.openmetadata.service.resources.apis;

import static org.openmetadata.service.util.EntityUtil.getEntityReference;

import org.openmetadata.schema.api.data.CreateAPIEndpoint;
import org.openmetadata.schema.entity.data.APIEndpoint;
import org.openmetadata.service.Entity;
import org.openmetadata.service.mapper.EntityMapper;

public class APIEndpointMapper implements EntityMapper<APIEndpoint, CreateAPIEndpoint> {
@Override
public APIEndpoint createToEntity(CreateAPIEndpoint create, String user) {
return copy(new APIEndpoint(), create, user)
.withApiCollection(getEntityReference(Entity.API_COLLCECTION, create.getApiCollection()))
.withRequestMethod(create.getRequestMethod())
.withEndpointURL(create.getEndpointURL())
.withRequestSchema(create.getRequestSchema())
.withResponseSchema(create.getResponseSchema())
.withSourceHash(create.getSourceHash());
}
}
Loading

0 comments on commit bab1ce4

Please sign in to comment.