Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DMP-4376 Add ARM RPO logs for Dynatrace #2337

Merged
merged 4 commits into from
Dec 5, 2024
Merged

DMP-4376 Add ARM RPO logs for Dynatrace #2337

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5fa6725
DMP-4376 Add ARM RPO logs for Dynatrace
karen-hedges Dec 4, 2024
126dc15
DMP-4376 Add ARM RPO logs for Dynatrace
karen-hedges Dec 3, 2024
d7d5c44
DMP-4376 Add ARM RPO logs for Dynatrace
karen-hedges Dec 3, 2024
d689272
DMP-4376 Add ARM RPO logs for Dynatrace
karen-hedges Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions ...java/uk/gov/hmcts/darts/task/runner/impl/ProcessE2EArmRpoPendingAutomatedTaskIntTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import uk.gov.hmcts.darts.arm.client.model.rpo.RecordManagementMatterResponse;
import uk.gov.hmcts.darts.arm.client.model.rpo.SaveBackgroundSearchResponse;
import uk.gov.hmcts.darts.arm.client.model.rpo.StorageAccountResponse;
import uk.gov.hmcts.darts.arm.exception.ArmRpoException;
import uk.gov.hmcts.darts.arm.service.ArmApiService;
import uk.gov.hmcts.darts.common.entity.ArmRpoExecutionDetailEntity;
import uk.gov.hmcts.darts.common.enums.ArmRpoStateEnum;
Expand All @@ -23,11 +22,8 @@
import java.util.Collections;
import java.util.List;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
Expand Down Expand Up @@ -112,10 +108,8 @@ void runTask_shouldThrowException_andSetExpectedExecutionDetailStateAndStatus_wh

// When
task.preRunTask();
ArmRpoException exception = assertThrows(ArmRpoException.class, () -> task.runTask());
assertThat(exception.getMessage(),
containsString("Failure during ARM RPO getRecordManagementMatter: ARM RPO API failed with status - 400 BAD_REQUEST"));

task.runTask();

// Then
List<ArmRpoExecutionDetailEntity> allExecutionDetails = dartsDatabase.getArmRpoExecutionDetailRepository()
.findAll();
Expand Down
72 changes: 43 additions & 29 deletions src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoPollServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@
import uk.gov.hmcts.darts.arm.service.ArmRpoService;
import uk.gov.hmcts.darts.authorisation.component.UserIdentity;
import uk.gov.hmcts.darts.common.entity.ArmRpoExecutionDetailEntity;
import uk.gov.hmcts.darts.common.entity.UserAccountEntity;
import uk.gov.hmcts.darts.common.service.FileOperationService;
import uk.gov.hmcts.darts.log.api.LogApi;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import static java.util.Objects.isNull;
Expand All @@ -35,29 +37,35 @@ public class ArmRpoPollServiceImpl implements ArmRpoPollService {
private final UserIdentity userIdentity;
private final FileOperationService fileOperationService;
private final ArmDataManagementConfiguration armDataManagementConfiguration;
private final LogApi logApi;

private List<File> tempProductionFiles;
private List<File> tempProductionFiles = new ArrayList<>();

private List<Integer> allowableFailedStates = null;
private List<Integer> allowableFailedStates = new ArrayList<>();


@Override
public void pollArmRpo(boolean isManualRun) {
log.info("Polling ARM RPO service - isManualRun: {}", isManualRun);
setupFailedStatuses();
Integer executionId = null;
tempProductionFiles = new ArrayList<>();
try {
var armRpoExecutionDetailEntity = getArmRpoExecutionDetailEntity(isManualRun);
if (isNull(armRpoExecutionDetailEntity)) {
log.warn("Unable to find armRpoExecutionDetailEntity to poll");
logApi.armRpoPollingFailed(executionId);
return;
}
executionId = armRpoExecutionDetailEntity.getId();

var bearerToken = armApiService.getArmBearerToken();
if (isNull(bearerToken)) {
log.warn("Unable to get bearer token to poll ARM RPO");
logApi.armRpoPollingFailed(executionId);
return;
}

var executionId = armRpoExecutionDetailEntity.getId();
var userAccount = userIdentity.getUserAccount();

// step to call ARM RPO API to get the extended searches by matter
Expand All @@ -78,17 +86,7 @@ public void pollArmRpo(boolean isManualRun) {
var productionOutputFiles = armRpoApi.getProductionOutputFiles(bearerToken, executionId, userAccount);

for (var productionExportFileId : productionOutputFiles) {
String productionExportFilename = generateTempProductionExportFilename(productionExportFileId);
// step to call ARM RPO API to download the production export file
var inputStream = armRpoApi.downloadProduction(bearerToken, executionId, productionExportFileId, userAccount);
log.info("About to save production export file to temp workspace {}", productionExportFilename);
Path tempProductionFile = fileOperationService.saveFileToTempWorkspace(
inputStream,
productionExportFilename,
armDataManagementConfiguration,
true
);
tempProductionFiles.add(tempProductionFile.toFile());
processProductionFiles(productionExportFileId, bearerToken, executionId, userAccount);
}
if (CollectionUtils.isNotEmpty(tempProductionFiles)) {
// step to call ARM RPO API to remove the production
Expand All @@ -101,41 +99,57 @@ public void pollArmRpo(boolean isManualRun) {
} else {
log.warn("Create export of production files is still in progress");
}

logApi.armRpoPollingSuccessful(executionId);
} catch (Exception e) {
log.error("Error while polling ARM RPO", e);
logApi.armRpoPollingFailed(executionId);
} finally {
try {
cleanUpTempFiles();
} catch (Exception e) {
log.error("Error while cleaning up ARM RPO polling service temp files", e);
}
cleanUpTempFiles();
}
}

private void processProductionFiles(String productionExportFileId, String bearerToken, Integer executionId,
UserAccountEntity userAccount) throws IOException {
String productionExportFilename = generateTempProductionExportFilename(productionExportFileId);
// step to call ARM RPO API to download the production export file
var inputStream = armRpoApi.downloadProduction(bearerToken, executionId, productionExportFileId, userAccount);
log.info("About to save production export file to temp workspace {}", productionExportFilename);
Path tempProductionFile = fileOperationService.saveFileToTempWorkspace(
inputStream,
productionExportFilename,
armDataManagementConfiguration,
true
);
tempProductionFiles.add(tempProductionFile.toFile());
}

private void setupFailedStatuses() {
if (CollectionUtils.isEmpty(allowableFailedStates)) {
allowableFailedStates = Collections.unmodifiableList(List.of(
allowableFailedStates = List.of(
ArmRpoHelper.getExtendedSearchesByMatterRpoState().getId(),
ArmRpoHelper.getMasterIndexFieldByRecordClassSchemaSecondaryRpoState().getId(),
ArmRpoHelper.createExportBasedOnSearchResultsTableRpoState().getId(),
ArmRpoHelper.getExtendedProductionsByMatterRpoState().getId(),
ArmRpoHelper.getProductionOutputFilesRpoState().getId(),
ArmRpoHelper.downloadProductionRpoState().getId(),
ArmRpoHelper.removeProductionRpoState().getId()
));
);
}
}

private void cleanUpTempFiles() {
for (var tempProductionFile : tempProductionFiles) {
if (tempProductionFile.exists()) {
if (tempProductionFile.delete()) {
log.info("Deleted temp production file {}", tempProductionFile.getName());
} else {
log.warn("Failed to delete temp production file {}", tempProductionFile.getName());
try {
for (var tempProductionFile : tempProductionFiles) {
if (tempProductionFile.exists()) {
if (tempProductionFile.delete()) {
log.info("Deleted temp production file {}", tempProductionFile.getName());
} else {
log.warn("Failed to delete temp production file {}", tempProductionFile.getName());
}
}
}
} catch (Exception e) {
log.error("Error while cleaning up ARM RPO polling service temp files", e);
}
}

Expand Down
73 changes: 40 additions & 33 deletions src/main/java/uk/gov/hmcts/darts/arm/service/impl/TriggerArmRpoSearchServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import uk.gov.hmcts.darts.arm.service.ArmRpoService;
import uk.gov.hmcts.darts.arm.service.TriggerArmRpoSearchService;
import uk.gov.hmcts.darts.authorisation.component.UserIdentity;
import uk.gov.hmcts.darts.log.api.LogApi;

@Slf4j
@Service
Expand All @@ -19,6 +20,7 @@ public class TriggerArmRpoSearchServiceImpl implements TriggerArmRpoSearchServic
private final ArmRpoService armRpoService;
private final ArmApiService armApiService;
private final UserIdentity userIdentity;
private final LogApi logApi;

/**
* This method integrates various ARM RPO API calls to ultimately trigger a search. The results of that search are then processed by another automated
Expand All @@ -31,50 +33,55 @@ public class TriggerArmRpoSearchServiceImpl implements TriggerArmRpoSearchServic
@Override
public void triggerArmRpoSearch() {
log.info("Triggering ARM RPO search flow...");
Integer executionId = null;
try {
var userAccountEntity = userIdentity.getUserAccount();

var userAccountEntity = userIdentity.getUserAccount();
executionId = armRpoService.createArmRpoExecutionDetailEntity(userAccountEntity).getId();

final Integer executionId = armRpoService.createArmRpoExecutionDetailEntity(userAccountEntity)
.getId();
// armBearerToken may be null, but we'll let the lower level service methods deal with that by handling the resultant HTTP exception
final String armBearerToken = armApiService.getArmBearerToken();

// armBearerToken may be null, but we'll let the lower level service methods deal with that by handling the resultant HTTP exception
final String armBearerToken = armApiService.getArmBearerToken();
armRpoApi.getRecordManagementMatter(armBearerToken,
executionId,
userAccountEntity);

armRpoApi.getRecordManagementMatter(armBearerToken,
executionId,
userAccountEntity);
// We expect getRecordManagementMatter() to populate the matter id as a side effect, so refresh the entity to get the updated value
final String matterId = armRpoService.getArmRpoExecutionDetailEntity(executionId)
.getMatterId();
armRpoApi.getIndexesByMatterId(armBearerToken,
executionId,
matterId,
userAccountEntity);

// We expect getRecordManagementMatter() to populate the matter id as a side effect, so refresh the entity to get the updated value
final String matterId = armRpoService.getArmRpoExecutionDetailEntity(executionId)
.getMatterId();
armRpoApi.getIndexesByMatterId(armBearerToken,
executionId,
matterId,
userAccountEntity);

armRpoApi.getStorageAccounts(armBearerToken,
executionId,
userAccountEntity);

armRpoApi.getProfileEntitlements(armBearerToken,
armRpoApi.getStorageAccounts(armBearerToken,
executionId,
userAccountEntity);

armRpoApi.getMasterIndexFieldByRecordClassSchema(armBearerToken,
executionId,
ArmRpoHelper.getMasterIndexFieldByRecordClassSchemaPrimaryRpoState(),
userAccountEntity);
armRpoApi.getProfileEntitlements(armBearerToken,
executionId,
userAccountEntity);

String searchName = armRpoApi.addAsyncSearch(armBearerToken,
executionId,
userAccountEntity);
armRpoApi.getMasterIndexFieldByRecordClassSchema(armBearerToken,
executionId,
ArmRpoHelper.getMasterIndexFieldByRecordClassSchemaPrimaryRpoState(),
userAccountEntity);

armRpoApi.saveBackgroundSearch(armBearerToken,
executionId,
searchName,
userAccountEntity);
String searchName = armRpoApi.addAsyncSearch(armBearerToken,
executionId,
userAccountEntity);

log.info("ARM RPO search flow completed successfully");
armRpoApi.saveBackgroundSearch(armBearerToken,
executionId,
searchName,
userAccountEntity);

log.info("ARM RPO search flow completed successfully");
logApi.armRpoSearchSuccessful(executionId);
} catch (Exception e) {
log.error("Error occurred during ARM RPO search flow", e);
logApi.armRpoSearchFailed(executionId);
}
}

}
8 changes: 8 additions & 0 deletions src/main/java/uk/gov/hmcts/darts/log/api/LogApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,12 @@ public interface LogApi {
void mediaDeleted(Integer mediaId);

void transcriptionDeleted(Integer transcriptionId);

void armRpoSearchSuccessful(Integer executionId);

void armRpoSearchFailed(Integer executionId);

void armRpoPollingSuccessful(Integer executionId);

void armRpoPollingFailed(Integer executionId);
}
21 changes: 21 additions & 0 deletions src/main/java/uk/gov/hmcts/darts/log/api/impl/LogApiImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,4 +162,25 @@ public void transcriptionDeleted(Integer transcriptionId) {
public void manualObfuscation(EventEntity eventEntity) {
eventLoggerService.manualObfuscation(eventEntity);
}

@Override
public void armRpoSearchSuccessful(Integer executionId) {
armLoggerService.armRpoSearchSuccessful(executionId);
}

@Override
public void armRpoSearchFailed(Integer executionId) {
armLoggerService.armRpoSearchFailed(executionId);
}

@Override
public void armRpoPollingSuccessful(Integer executionId) {
armLoggerService.armRpoPollingSuccessful(executionId);
}

@Override
public void armRpoPollingFailed(Integer executionId) {
armLoggerService.armRpoPollingFailed(executionId);
}

}
7 changes: 7 additions & 0 deletions src/main/java/uk/gov/hmcts/darts/log/service/ArmLoggerService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,11 @@ public interface ArmLoggerService {

void archiveToArmFailed(Integer eodId);

void armRpoSearchSuccessful(Integer executionId);

void armRpoSearchFailed(Integer executionId);

void armRpoPollingSuccessful(Integer executionId);

void armRpoPollingFailed(Integer executionId);
}
33 changes: 27 additions & 6 deletions src/main/java/uk/gov/hmcts/darts/log/service/impl/ArmLoggerServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@
public class ArmLoggerServiceImpl implements ArmLoggerService {

private static final String ARM_PUSH_SUCCESSFUL = "Successfully pushed object to ARM dropzone: eod_id={}";

private static final String ARM_PUSHED_FAILED = "Failed to push object to ARM dropzone: eod_id={}";

private static final String ARCHIVE_TO_ARM_SUCCESSFUL = "Successfully archived object to ARM: eod_id={}";

private static final String ARCHIVE_TO_ARM_FAILED = "Failed to archive object to ARM: eod_id={}";


private static final String ARM_RPO_SEARCH_SUCCESSFULLY_COMPLETED_FOR_EXECUTION_ID = "ARM RPO Search - Successfully completed for execution Id = {}";
private static final String ARM_RPO_SEARCH_FAILED_FOR_EXECUTION_ID = "ARM RPO Search - Failed for execution Id = {}";
private static final String ARM_RPO_POLLING_SUCCESSFULLY_COMPLETED_FOR_EXECUTION_ID = "ARM RPO Polling - Successfully completed for execution Id = {}";
private static final String ARM_RPO_POLLING_FAILED_FOR_EXECUTION_ID = "ARM RPO Polling - Failed for execution Id = {}";

@Override
public void armPushSuccessful(Integer eodId) {
log.info(ARM_PUSH_SUCCESSFUL, eodId);
Expand All @@ -36,6 +36,27 @@ public void archiveToArmSuccessful(Integer eodId) {

@Override
public void archiveToArmFailed(Integer eodId) {
log.info(ARCHIVE_TO_ARM_FAILED, eodId);
log.error(ARCHIVE_TO_ARM_FAILED, eodId);
}

@Override
public void armRpoSearchSuccessful(Integer executionId) {
log.info(ARM_RPO_SEARCH_SUCCESSFULLY_COMPLETED_FOR_EXECUTION_ID, executionId);
}

@Override
public void armRpoSearchFailed(Integer executionId) {
log.error(ARM_RPO_SEARCH_FAILED_FOR_EXECUTION_ID, executionId);
}

@Override
public void armRpoPollingSuccessful(Integer executionId) {
log.info(ARM_RPO_POLLING_SUCCESSFULLY_COMPLETED_FOR_EXECUTION_ID, executionId);
}

@Override
public void armRpoPollingFailed(Integer executionId) {
log.error(ARM_RPO_POLLING_FAILED_FOR_EXECUTION_ID, executionId);
}

}
Loading
Loading