Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: test case result reindex performance #18723

Merged
merged 2 commits into from
Nov 21, 2024
Merged

fix: test case result reindex performance #18723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
42d5c7d
fix: test case reindex performance
TeddyCr Nov 21, 2024
ba96b24
Merge branch 'main' into MINOR-Optimize-tcr-index
TeddyCr Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ private void deleteAllTestCaseResults(String fqn) {

@SneakyThrows
private TestCaseResult getTestCaseResult(TestCase testCase) {
TestCaseResult testCaseResult;
TestCaseResult testCaseResult = null;
if (testCase.getTestCaseResult() != null) {
// we'll return the saved state if it exists otherwise we'll fetch it from the database
// Should be the case if listing from the search repo. as the test case result
Expand All @@ -387,8 +387,6 @@ private TestCaseResult getTestCaseResult(TestCase testCase) {
LOG.debug(
"Error fetching test case result from search. Fetching from test case results from database",
e);
testCaseResult =
timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
}
if (nullOrEmpty(testCaseResult)) {
testCaseResult =
Expand Down
15 changes: 7 additions & 8 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,11 @@ public TestSuiteRepository() {
public void setFields(TestSuite entity, EntityUtil.Fields fields) {
entity.setPipelines(
fields.contains("pipelines") ? getIngestionPipelines(entity) : entity.getPipelines());
entity.withTests(fields.contains(UPDATE_FIELDS) ? getTestCases(entity) : entity.getTests());
entity.withTestCaseResultSummary(getResultSummary(entity.getId()));
entity.setTests(fields.contains(UPDATE_FIELDS) ? getTestCases(entity) : entity.getTests());
entity.setTestCaseResultSummary(
fields.contains("summary")
? getResultSummary(entity.getId())
: entity.getTestCaseResultSummary());
entity.setSummary(
fields.contains("summary")
? getTestSummary(entity.getTestCaseResultSummary())
Expand Down Expand Up @@ -337,7 +340,7 @@ protected void postCreate(TestSuite entity) {
@SneakyThrows
private List<ResultSummary> getResultSummary(UUID testSuiteId) {
List<ResultSummary> resultSummaries = new ArrayList<>();
ResultList<TestCaseResult> latestTestCaseResultResults;
ResultList<TestCaseResult> latestTestCaseResultResults = null;
String groupBy = "testCaseFQN.keyword";
SearchListFilter searchListFilter = new SearchListFilter();
searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
Expand All @@ -348,16 +351,12 @@ private List<ResultSummary> getResultSummary(UUID testSuiteId) {
entityTimeSeriesRepository.listLatestFromSearch(
EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
} catch (Exception e) {
// Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
// to database
LOG.debug(
"Error fetching test case result from search. Fetching from test case results from database",
e);
latestTestCaseResultResults =
entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
}

if (nullOrEmpty(latestTestCaseResultResults.getData())) {
if (latestTestCaseResultResults == null || nullOrEmpty(latestTestCaseResultResults.getData())) {
latestTestCaseResultResults =
entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
}
Expand Down
13 changes: 11 additions & 2 deletions ...ta-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,11 @@ public void removeNonIndexableFields(Map<String, Object> esDoc) {
@Override
public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> esDoc) {
TestCase testCase =
Entity.getEntityByName(Entity.TEST_CASE, testCaseResult.getTestCaseFQN(), "*", Include.ALL);
Entity.getEntityByName(
Entity.TEST_CASE,
testCaseResult.getTestCaseFQN(),
"owners,testSuites,testDefinition,domain,tags",
Include.ALL);
TestDefinition testDefinition =
Entity.getEntityByName(
Entity.TEST_DEFINITION,
Expand Down Expand Up @@ -80,7 +84,12 @@ private void setParentRelationships(TestCase testCase, Map<String, Object> esDoc

private void setTableEntityParentRelations(
MessageParser.EntityLink entityLink, Map<String, Object> esDoc) {
Table table = Entity.getEntityByName(Entity.TABLE, entityLink.getEntityFQN(), "*", Include.ALL);
Table table =
Entity.getEntityByName(
Entity.TABLE,
entityLink.getEntityFQN(),
"owners,columns,tags,followers,schemaDefinition,dataModel,extension,domain,dataProducts",
Include.ALL);
EntityReference databaseSchemaReference = table.getDatabaseSchema();
EntityReference databaseReference = table.getDatabase();
EntityReference serviceReference = table.getService();
Expand Down
Loading