Skip to content

Commit

Permalink
DMP-4256 CR and IL files are not getting deleted (#2342)
Browse files Browse the repository at this point in the history
  • Loading branch information
@karen-hedges
karen-hedges authored Dec 5, 2024
1 parent fc81390 commit f9b8143
Show file tree
Hide file tree
Showing 3 changed files with 170 additions and 8 deletions.
19 changes: 14 additions & 5 deletions bin/generateBatchArmResponses.sh
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
echo "This script generates the ARM pull response files. These can be used for testing where ARM returns a response for an object such as media, transcription document, annotation document or case document to be stored in the ARM storage"

uuid1=$(uuidgen)
uuid1=$(uuidgen | tr -d '-')

echo "uuid 1: $uuid1"
echo " "
Expand Down Expand Up @@ -45,7 +45,7 @@ addNewResponses () {
fi

#${parameter//pattern/string}
uiFileContentsParam="{\"operation\": \"upload_file\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}"
uiFileContentsParam="{\"operation\": \"upload_file\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\\\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\\\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}"
uiFileContents=${uiFileContentsParam//EODID/$eodid}
echo "$uiFileContents"
echo $uiFileContents >> $iuFilename
Expand Down Expand Up @@ -92,12 +92,15 @@ addNewResponses () {

if [ "$option" == "1" ] || [ "$option" == "2" ]
then
crFilename="UUID1_UUID2_1_cr.rsp"
statusCode=1

crFilename="UUID1_UUID2_STATUSCODE_cr.rsp"
crFilename=${crFilename//STATUSCODE/$statusCode}
crFilename=${crFilename//UUID1/$uuid1}
crFilename=${crFilename//UUID2/$uuid2}
echo "CR filename: $crFilename"

crFileContentsParam="{\"operation\": \"create_record\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}"
crFileContentsParam="{\"operation\": \"create_record\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\\\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\\\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\\\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}"
crFileContents=${crFileContentsParam//EODID/$eodid}
echo "$crFileContents"
echo $crFileContents > $crFilename
Expand Down Expand Up @@ -142,7 +145,13 @@ addNewResponses () {
exit 1
fi

ufFilename="UUID1_UUID5_0_uf.rsp"
statusCode=1
if [ "$option" == "3" ]
then
statusCode=0
fi
ufFilename="UUID1_UUID5_STATUSCODE_uf.rsp"
ufFilename=${ufFilename//STATUSCODE/$statusCode}
ufFilename=${ufFilename//UUID1/$uuid1}
ufFilename=${ufFilename//UUID5/$uuid5}
echo "UF filename : $ilFilename"
Expand Down
93 changes: 93 additions & 0 deletions ...Test/java/uk/gov/hmcts/darts/arm/service/AbstractArmBatchProcessResponseFilesIntTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,93 @@ void batchProcessResponseFiles_WithInvalidLineFileAndCreateRecordFileSuccess() t

}

@Test
void batchProcessResponseFiles_successful_withUploadFileAndCreateRecordFile() throws IOException {

// given
HearingEntity hearing = PersistableFactory.getHearingTestData().someMinimal();

OffsetDateTime startTime = OffsetDateTime.parse(T_13_00_00_Z);
OffsetDateTime endTime = OffsetDateTime.parse(T_13_45_00_Z);
MediaEntity media1 = createMediaEntity(hearing, startTime, endTime, 1);
when(currentTimeHelper.currentOffsetDateTime()).thenReturn(endTime);

String manifest1Uuid = UUID.randomUUID().toString();
String manifestFile1 = prefix() + "_" + manifest1Uuid + ".a360";

ExternalObjectDirectoryEntity armEod1 = PersistableFactory.getExternalObjectDirectoryTestData().someMinimalBuilder()
.media(media1).status(dartsDatabase.getObjectRecordStatusEntity(ARM_DROP_ZONE))
.externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM)).externalLocation(UUID.randomUUID()).build();
armEod1.setTransferAttempts(1);
armEod1.setVerificationAttempts(1);
armEod1.setManifestFile(manifestFile1);
armEod1.setChecksum("7017013d05bcc5032e142049081821d6");
armEod1 = dartsPersistence.save(armEod1);

List<String> blobNamesAndPaths = new ArrayList<>();
String blobNameAndPath1 = String.format("dropzone/DARTS/response/%s_%s_6a374f19a9ce7dc9cc480ea8d4eca0fb_1_iu.rsp", prefix(), manifest1Uuid);
blobNamesAndPaths.add(blobNameAndPath1);

ContinuationTokenBlobs continuationTokenBlobs = ContinuationTokenBlobs.builder()
.blobNamesAndPaths(blobNamesAndPaths)
.build();

when(armDataManagementApi.listResponseBlobsUsingMarker(prefix(), BATCH_SIZE, continuationToken)).thenReturn(continuationTokenBlobs);
String hashcode1 = "6a374f19a9ce7dc9cc480ea8d4eca0fb";
String createRecordFilename1 = String.format("%s_a17b9015-e6ad-77c5-8d1e-13259aae1895_1_cr.rsp", hashcode1);
String uploadFileFilename1 = String.format("%s_04e6bc3b-952a-79b6-8362-13259aae1895_1_uf.rsp", hashcode1);

List<String> hashcodeResponses = List.of(createRecordFilename1, uploadFileFilename1);

when(armDataManagementApi.listResponseBlobs(hashcode1)).thenReturn(hashcodeResponses);

String createRecordFileTest1 = "tests/arm/service/ArmBatchResponseFilesProcessorTest/ValidResponses/CreateRecord.rsp";
String validUploadFileTest1 = "tests/arm/service/ArmBatchResponseFilesProcessorTest/ValidResponses/UploadFile.rsp";

BinaryData createRecordBinaryDataTest1 = convertStringToBinaryData(getCreateRecordFileContents(createRecordFileTest1, armEod1.getId()));
BinaryData uploadFileBinaryDataTest1 = convertStringToBinaryData(getUploadFileContents(validUploadFileTest1, armEod1.getId(), media1.getChecksum()));


when(armDataManagementApi.getBlobData(createRecordFilename1)).thenReturn(createRecordBinaryDataTest1);
when(armDataManagementApi.getBlobData(uploadFileFilename1)).thenReturn(uploadFileBinaryDataTest1);

when(armDataManagementApi.deleteBlobData(createRecordFilename1)).thenReturn(true);
when(armDataManagementApi.deleteBlobData(uploadFileFilename1)).thenReturn(true);

String fileLocation = tempDirectory.getAbsolutePath();
when(armDataManagementConfiguration.getTempBlobWorkspace()).thenReturn(fileLocation);
when(armDataManagementConfiguration.getContinuationTokenDuration()).thenReturn("PT1M");
when(armDataManagementConfiguration.getManifestFilePrefix()).thenReturn(prefix());
when(armDataManagementConfiguration.getFileExtension()).thenReturn("a360");

// when
armBatchProcessResponseFiles.processResponseFiles(BATCH_SIZE);

// then
List<ExternalObjectDirectoryEntity> foundMediaList = dartsDatabase.getExternalObjectDirectoryRepository()
.findByMediaAndExternalLocationType(media1, dartsDatabase.getExternalLocationTypeEntity(ARM));

assertEquals(1, foundMediaList.size());
ExternalObjectDirectoryEntity foundMedia = foundMediaList.getFirst();
assertEquals(ARM_RPO_PENDING.getId(), foundMedia.getStatus().getId());
assertEquals(1, foundMedia.getVerificationAttempts());
assertNotNull(foundMedia.getDataIngestionTs());
assertTrue(foundMedia.isResponseCleaned());

verify(armDataManagementApi).listResponseBlobsUsingMarker(prefix(), BATCH_SIZE, continuationToken);
verify(armDataManagementApi).listResponseBlobs(hashcode1);

verify(armDataManagementApi).getBlobData(createRecordFilename1);
verify(armDataManagementApi).getBlobData(uploadFileFilename1);

verify(armDataManagementApi).deleteBlobData(createRecordFilename1);
verify(armDataManagementApi).deleteBlobData(uploadFileFilename1);
verify(armDataManagementApi).deleteBlobData(blobNameAndPath1);

verifyNoMoreInteractions(armDataManagementApi);

}

@Test
void batchProcessResponseFiles_With3InvalidLineFilesCausingFailure() throws IOException {

Expand Down Expand Up @@ -2281,6 +2368,12 @@ protected BinaryData convertStringToBinaryData(String contents) {
return BinaryData.fromString(contents);
}

protected String getInputUploadFileContents(String uploadFilename, Integer externalObjectDirectoryId) throws IOException {
String expectedResponse = getContentsFromFile(uploadFilename);
expectedResponse = expectedResponse.replaceAll("<EODID>", String.valueOf(externalObjectDirectoryId));
return expectedResponse;
}

protected String getInvalidLineFileContents(String invalidLineFilename, Integer externalObjectDirectoryId) throws IOException {
String expectedResponse = getContentsFromFile(invalidLineFilename);
expectedResponse = expectedResponse.replaceAll("<EODID>", String.valueOf(externalObjectDirectoryId));
Expand Down
66 changes: 63 additions & 3 deletions src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ && nonNull(armResponseBatchData.getUploadFileFilenameProcessor())) {
deleteResponseBlobs(armResponseBatchData);
} else {
log.info("Unable to find response files for external object {}", armResponseBatchData.getExternalObjectDirectoryId());
logResponsesFound(armResponseBatchData);
try {
ExternalObjectDirectoryEntity externalObjectDirectory =
getExternalObjectDirectoryEntity(armResponseBatchData.getExternalObjectDirectoryId());
Expand All @@ -328,6 +329,35 @@ && nonNull(armResponseBatchData.getUploadFileFilenameProcessor())) {
);
}

private void logResponsesFound(ArmResponseBatchData armResponseBatchData) {

int eodId = armResponseBatchData.getExternalObjectDirectoryId();
List<InvalidLineFileFilenameProcessor> invalidLineFileFilenameProcessors = armResponseBatchData.getInvalidLineFileFilenameProcessors();
CreateRecordFilenameProcessor createRecordFilenameProcessor = armResponseBatchData.getCreateRecordFilenameProcessor();
UploadFileFilenameProcessor uploadFileFilenameProcessor = armResponseBatchData.getUploadFileFilenameProcessor();

log.info("Found ARM responses for external object directory ID {} with {} invalid line files, {} create record files and {} upload files",
eodId,
invalidLineFileFilenameProcessors.size(),
nonNull(createRecordFilenameProcessor) ? 1 : 0,
nonNull(uploadFileFilenameProcessor) ? 1 : 0);
if (nonNull(createRecordFilenameProcessor)) {
log.info("Found eod {} with create record file: {}", eodId,
createRecordFilenameProcessor.getCreateRecordFilenameAndPath());
}
if (nonNull(uploadFileFilenameProcessor)) {
log.info("Found eod {} with upload file: {}", eodId,
uploadFileFilenameProcessor.getUploadFileFilenameAndPath());
}
if (CollectionUtils.isNotEmpty(invalidLineFileFilenameProcessors)) {
invalidLineFileFilenameProcessors.forEach(
invalidLineFileFilenameProcessor ->
log.info("Found eod {} with invalid line file: {}", eodId,
invalidLineFileFilenameProcessor.getInvalidLineFileFilenameAndPath())
);
}
}

private void processMultipleInvalidLineFiles(ArmResponseBatchData armResponseBatchData) {
try {
ExternalObjectDirectoryEntity externalObjectDirectory = getExternalObjectDirectoryEntity(armResponseBatchData.getExternalObjectDirectoryId());
Expand Down Expand Up @@ -445,7 +475,7 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil

if (nonNull(jsonPath) && jsonPath.toFile().exists()) {
logResponseFileContents(jsonPath);
ArmResponseCreateRecord armResponseCreateRecord = objectMapper.readValue(jsonPath.toFile(), ArmResponseCreateRecord.class);
ArmResponseCreateRecord armResponseCreateRecord = getResponseCreateRecordOrDelete(jsonPath);
UploadNewFileRecord uploadNewFileRecord = readInputJson(armResponseCreateRecord.getInput());
if (nonNull(uploadNewFileRecord)) {
if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) {
Expand Down Expand Up @@ -476,6 +506,16 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil

}

private ArmResponseCreateRecord getResponseCreateRecordOrDelete(Path jsonPath) throws IOException {
try {
return objectMapper.readValue(jsonPath.toFile(), ArmResponseCreateRecord.class);
} catch (Exception e) {
log.error("Unable to read ARM response create record file {} - About to delete ", jsonPath.toFile().getAbsoluteFile(), e);
deleteResponseBlobs(List.of(jsonPath.toFile().getAbsolutePath()));
throw e;
}
}

private void processUploadResponseFiles(List<UploadFileFilenameProcessor> uploadFileResponses, ArmBatchResponses armBatchResponses) {
for (UploadFileFilenameProcessor uploadFileFilenameProcessor : uploadFileResponses) {
try {
Expand Down Expand Up @@ -503,7 +543,7 @@ private void readUploadFile(BinaryData uploadFileBinary, UploadFileFilenameProce

if (jsonPath.toFile().exists()) {
logResponseFileContents(jsonPath);
ArmResponseUploadFileRecord armResponseUploadFileRecord = objectMapper.readValue(jsonPath.toFile(), ArmResponseUploadFileRecord.class);
ArmResponseUploadFileRecord armResponseUploadFileRecord = getResponseUploadFileRecordOrDelete(jsonPath);
UploadNewFileRecord uploadNewFileRecord = readInputJson(armResponseUploadFileRecord.getInput());
if (nonNull(uploadNewFileRecord)) {
if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) {
Expand Down Expand Up @@ -534,6 +574,16 @@ private void readUploadFile(BinaryData uploadFileBinary, UploadFileFilenameProce
}
}

private ArmResponseUploadFileRecord getResponseUploadFileRecordOrDelete(Path jsonPath) throws IOException {
try {
return objectMapper.readValue(jsonPath.toFile(), ArmResponseUploadFileRecord.class);
} catch (Exception e) {
log.error("Unable to read ARM response upload file {} - About to delete ", jsonPath.toFile().getAbsoluteFile(), e);
deleteResponseBlobs(List.of(jsonPath.toFile().getAbsolutePath()));
throw e;
}
}

private void logResponseFileContents(Path jsonPath) {
try {
String contents = FileUtils.readFileToString(jsonPath.toFile(), UTF_8);
Expand Down Expand Up @@ -713,7 +763,7 @@ private void readInvalidLineFile(BinaryData invalidLineFileBinary, InvalidLineFi

if (jsonPath.toFile().exists()) {
logResponseFileContents(jsonPath);
ArmResponseInvalidLineRecord armResponseInvalidLineRecord = objectMapper.readValue(jsonPath.toFile(), ArmResponseInvalidLineRecord.class);
ArmResponseInvalidLineRecord armResponseInvalidLineRecord = getResponseInvalidLineRecordOrDelete(jsonPath);
String input = armResponseInvalidLineRecord.getInput();
UploadNewFileRecord uploadNewFileRecord = readInputJson(input);
if (nonNull(uploadNewFileRecord)) {
Expand Down Expand Up @@ -748,6 +798,16 @@ private void readInvalidLineFile(BinaryData invalidLineFileBinary, InvalidLineFi
}
}

private ArmResponseInvalidLineRecord getResponseInvalidLineRecordOrDelete(Path jsonPath) throws IOException {
try {
return objectMapper.readValue(jsonPath.toFile(), ArmResponseInvalidLineRecord.class);
} catch (Exception e) {
log.error("Unable to read ARM response invalid line file {} - About to delete ", jsonPath.toFile().getAbsoluteFile(), e);
deleteResponseBlobs(List.of(jsonPath.toFile().getAbsolutePath()));
throw e;
}
}

private void processInvalidLineFile(int externalObjectDirectoryId, InvalidLineFileFilenameProcessor invalidLineFileFilenameProcessor,
ArmResponseInvalidLineRecord armResponseInvalidLineRecord,
CreateRecordFilenameProcessor createRecordFilenameProcessor,
Expand Down

0 comments on commit f9b8143

Please sign in to comment.