From f9b8143358c9db5c559849fa5e7196a2e79f9f41 Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:04:31 +0000 Subject: [PATCH] DMP-4256 CR and IL files are not getting deleted (#2342) --- bin/generateBatchArmResponses.sh | 19 +++- ...ctArmBatchProcessResponseFilesIntTest.java | 93 +++++++++++++++++++ .../AbstractArmBatchProcessResponseFiles.java | 66 ++++++++++++- 3 files changed, 170 insertions(+), 8 deletions(-) diff --git a/bin/generateBatchArmResponses.sh b/bin/generateBatchArmResponses.sh index 649e3e421b..3be4825338 100755 --- a/bin/generateBatchArmResponses.sh +++ b/bin/generateBatchArmResponses.sh @@ -1,7 +1,7 @@ #!/bin/bash echo "This script generates the ARM pull response files. These can be used for testing where ARM returns a response for an object such as media, transcription document, annotation document or case document to be stored in the ARM storage" -uuid1=$(uuidgen) +uuid1=$(uuidgen | tr -d '-') echo "uuid 1: $uuid1" echo " " @@ -45,7 +45,7 @@ addNewResponses () { fi #${parameter//pattern/string} - uiFileContentsParam="{\"operation\": \"upload_file\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}" + uiFileContentsParam="{\"operation\": \"upload_file\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\\\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\\\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}" uiFileContents=${uiFileContentsParam//EODID/$eodid} echo "$uiFileContents" echo $uiFileContents >> $iuFilename @@ -92,12 +92,15 @@ addNewResponses () { if [ "$option" == "1" ] || [ "$option" == "2" ] then - crFilename="UUID1_UUID2_1_cr.rsp" + statusCode=1 + + crFilename="UUID1_UUID2_STATUSCODE_cr.rsp" + crFilename=${crFilename//STATUSCODE/$statusCode} crFilename=${crFilename//UUID1/$uuid1} crFilename=${crFilename//UUID2/$uuid2} echo "CR filename: $crFilename" - crFileContentsParam="{\"operation\": \"create_record\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}" + crFileContentsParam="{\"operation\": \"create_record\", \"transaction_id\": \"2d1c7f6f-224e-768e-a274-41af570e6502\", \"relation_id\": \"EODID\", \"a360_record_id\": \"1cf976c7-cedd-703f-ab70-01588bd56d50\", \"process_time\": \"2023-07-11T11:39:26.790000\", \"status\": 1, \"input\": \"{\\\"operation\\\": \\\"create_record\\\",\\\"relation_id\\\": \\\"EODID\\\",\\\"record_metadata\\\": {\\\"record_class\\\": \\\"A360TEST\\\",\\\"publisher\\\": \\\"A360\\\",\\\"recordDate\\\": \\\"2016-11-22T11:39:30Z\\\",\\\"region\\\": \\\"GBR\\\",\\\"title\\\": \\\"A360230711_TestIngestion_2\\\"}}\", \"exception_description\": null, \"errorStatus1\": null}" crFileContents=${crFileContentsParam//EODID/$eodid} echo "$crFileContents" echo $crFileContents > $crFilename @@ -142,7 +145,13 @@ addNewResponses () { exit 1 fi - ufFilename="UUID1_UUID5_0_uf.rsp" + statusCode=1 + if [ "$option" == "3" ] + then + statusCode=0 + fi + ufFilename="UUID1_UUID5_STATUSCODE_uf.rsp" + ufFilename=${ufFilename//STATUSCODE/$statusCode} ufFilename=${ufFilename//UUID1/$uuid1} ufFilename=${ufFilename//UUID5/$uuid5} echo "UF filename : $ilFilename" diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/AbstractArmBatchProcessResponseFilesIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/AbstractArmBatchProcessResponseFilesIntTest.java index dafcaaf954..03dc581b02 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/AbstractArmBatchProcessResponseFilesIntTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/AbstractArmBatchProcessResponseFilesIntTest.java @@ -459,6 +459,93 @@ void batchProcessResponseFiles_WithInvalidLineFileAndCreateRecordFileSuccess() t } + @Test + void batchProcessResponseFiles_successful_withUploadFileAndCreateRecordFile() throws IOException { + + // given + HearingEntity hearing = PersistableFactory.getHearingTestData().someMinimal(); + + OffsetDateTime startTime = OffsetDateTime.parse(T_13_00_00_Z); + OffsetDateTime endTime = OffsetDateTime.parse(T_13_45_00_Z); + MediaEntity media1 = createMediaEntity(hearing, startTime, endTime, 1); + when(currentTimeHelper.currentOffsetDateTime()).thenReturn(endTime); + + String manifest1Uuid = UUID.randomUUID().toString(); + String manifestFile1 = prefix() + "_" + manifest1Uuid + ".a360"; + + ExternalObjectDirectoryEntity armEod1 = PersistableFactory.getExternalObjectDirectoryTestData().someMinimalBuilder() + .media(media1).status(dartsDatabase.getObjectRecordStatusEntity(ARM_DROP_ZONE)) + .externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM)).externalLocation(UUID.randomUUID()).build(); + armEod1.setTransferAttempts(1); + armEod1.setVerificationAttempts(1); + armEod1.setManifestFile(manifestFile1); + armEod1.setChecksum("7017013d05bcc5032e142049081821d6"); + armEod1 = dartsPersistence.save(armEod1); + + List blobNamesAndPaths = new ArrayList<>(); + String blobNameAndPath1 = String.format("dropzone/DARTS/response/%s_%s_6a374f19a9ce7dc9cc480ea8d4eca0fb_1_iu.rsp", prefix(), manifest1Uuid); + blobNamesAndPaths.add(blobNameAndPath1); + + ContinuationTokenBlobs continuationTokenBlobs = ContinuationTokenBlobs.builder() + .blobNamesAndPaths(blobNamesAndPaths) + .build(); + + when(armDataManagementApi.listResponseBlobsUsingMarker(prefix(), BATCH_SIZE, continuationToken)).thenReturn(continuationTokenBlobs); + String hashcode1 = "6a374f19a9ce7dc9cc480ea8d4eca0fb"; + String createRecordFilename1 = String.format("%s_a17b9015-e6ad-77c5-8d1e-13259aae1895_1_cr.rsp", hashcode1); + String uploadFileFilename1 = String.format("%s_04e6bc3b-952a-79b6-8362-13259aae1895_1_uf.rsp", hashcode1); + + List hashcodeResponses = List.of(createRecordFilename1, uploadFileFilename1); + + when(armDataManagementApi.listResponseBlobs(hashcode1)).thenReturn(hashcodeResponses); + + String createRecordFileTest1 = "tests/arm/service/ArmBatchResponseFilesProcessorTest/ValidResponses/CreateRecord.rsp"; + String validUploadFileTest1 = "tests/arm/service/ArmBatchResponseFilesProcessorTest/ValidResponses/UploadFile.rsp"; + + BinaryData createRecordBinaryDataTest1 = convertStringToBinaryData(getCreateRecordFileContents(createRecordFileTest1, armEod1.getId())); + BinaryData uploadFileBinaryDataTest1 = convertStringToBinaryData(getUploadFileContents(validUploadFileTest1, armEod1.getId(), media1.getChecksum())); + + + when(armDataManagementApi.getBlobData(createRecordFilename1)).thenReturn(createRecordBinaryDataTest1); + when(armDataManagementApi.getBlobData(uploadFileFilename1)).thenReturn(uploadFileBinaryDataTest1); + + when(armDataManagementApi.deleteBlobData(createRecordFilename1)).thenReturn(true); + when(armDataManagementApi.deleteBlobData(uploadFileFilename1)).thenReturn(true); + + String fileLocation = tempDirectory.getAbsolutePath(); + when(armDataManagementConfiguration.getTempBlobWorkspace()).thenReturn(fileLocation); + when(armDataManagementConfiguration.getContinuationTokenDuration()).thenReturn("PT1M"); + when(armDataManagementConfiguration.getManifestFilePrefix()).thenReturn(prefix()); + when(armDataManagementConfiguration.getFileExtension()).thenReturn("a360"); + + // when + armBatchProcessResponseFiles.processResponseFiles(BATCH_SIZE); + + // then + List foundMediaList = dartsDatabase.getExternalObjectDirectoryRepository() + .findByMediaAndExternalLocationType(media1, dartsDatabase.getExternalLocationTypeEntity(ARM)); + + assertEquals(1, foundMediaList.size()); + ExternalObjectDirectoryEntity foundMedia = foundMediaList.getFirst(); + assertEquals(ARM_RPO_PENDING.getId(), foundMedia.getStatus().getId()); + assertEquals(1, foundMedia.getVerificationAttempts()); + assertNotNull(foundMedia.getDataIngestionTs()); + assertTrue(foundMedia.isResponseCleaned()); + + verify(armDataManagementApi).listResponseBlobsUsingMarker(prefix(), BATCH_SIZE, continuationToken); + verify(armDataManagementApi).listResponseBlobs(hashcode1); + + verify(armDataManagementApi).getBlobData(createRecordFilename1); + verify(armDataManagementApi).getBlobData(uploadFileFilename1); + + verify(armDataManagementApi).deleteBlobData(createRecordFilename1); + verify(armDataManagementApi).deleteBlobData(uploadFileFilename1); + verify(armDataManagementApi).deleteBlobData(blobNameAndPath1); + + verifyNoMoreInteractions(armDataManagementApi); + + } + @Test void batchProcessResponseFiles_With3InvalidLineFilesCausingFailure() throws IOException { @@ -2281,6 +2368,12 @@ protected BinaryData convertStringToBinaryData(String contents) { return BinaryData.fromString(contents); } + protected String getInputUploadFileContents(String uploadFilename, Integer externalObjectDirectoryId) throws IOException { + String expectedResponse = getContentsFromFile(uploadFilename); + expectedResponse = expectedResponse.replaceAll("", String.valueOf(externalObjectDirectoryId)); + return expectedResponse; + } + protected String getInvalidLineFileContents(String invalidLineFilename, Integer externalObjectDirectoryId) throws IOException { String expectedResponse = getContentsFromFile(invalidLineFilename); expectedResponse = expectedResponse.replaceAll("", String.valueOf(externalObjectDirectoryId)); diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java index ecee2cdf4a..32586fc94b 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java @@ -314,6 +314,7 @@ && nonNull(armResponseBatchData.getUploadFileFilenameProcessor())) { deleteResponseBlobs(armResponseBatchData); } else { log.info("Unable to find response files for external object {}", armResponseBatchData.getExternalObjectDirectoryId()); + logResponsesFound(armResponseBatchData); try { ExternalObjectDirectoryEntity externalObjectDirectory = getExternalObjectDirectoryEntity(armResponseBatchData.getExternalObjectDirectoryId()); @@ -328,6 +329,35 @@ && nonNull(armResponseBatchData.getUploadFileFilenameProcessor())) { ); } + private void logResponsesFound(ArmResponseBatchData armResponseBatchData) { + + int eodId = armResponseBatchData.getExternalObjectDirectoryId(); + List invalidLineFileFilenameProcessors = armResponseBatchData.getInvalidLineFileFilenameProcessors(); + CreateRecordFilenameProcessor createRecordFilenameProcessor = armResponseBatchData.getCreateRecordFilenameProcessor(); + UploadFileFilenameProcessor uploadFileFilenameProcessor = armResponseBatchData.getUploadFileFilenameProcessor(); + + log.info("Found ARM responses for external object directory ID {} with {} invalid line files, {} create record files and {} upload files", + eodId, + invalidLineFileFilenameProcessors.size(), + nonNull(createRecordFilenameProcessor) ? 1 : 0, + nonNull(uploadFileFilenameProcessor) ? 1 : 0); + if (nonNull(createRecordFilenameProcessor)) { + log.info("Found eod {} with create record file: {}", eodId, + createRecordFilenameProcessor.getCreateRecordFilenameAndPath()); + } + if (nonNull(uploadFileFilenameProcessor)) { + log.info("Found eod {} with upload file: {}", eodId, + uploadFileFilenameProcessor.getUploadFileFilenameAndPath()); + } + if (CollectionUtils.isNotEmpty(invalidLineFileFilenameProcessors)) { + invalidLineFileFilenameProcessors.forEach( + invalidLineFileFilenameProcessor -> + log.info("Found eod {} with invalid line file: {}", eodId, + invalidLineFileFilenameProcessor.getInvalidLineFileFilenameAndPath()) + ); + } + } + private void processMultipleInvalidLineFiles(ArmResponseBatchData armResponseBatchData) { try { ExternalObjectDirectoryEntity externalObjectDirectory = getExternalObjectDirectoryEntity(armResponseBatchData.getExternalObjectDirectoryId()); @@ -445,7 +475,7 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil if (nonNull(jsonPath) && jsonPath.toFile().exists()) { logResponseFileContents(jsonPath); - ArmResponseCreateRecord armResponseCreateRecord = objectMapper.readValue(jsonPath.toFile(), ArmResponseCreateRecord.class); + ArmResponseCreateRecord armResponseCreateRecord = getResponseCreateRecordOrDelete(jsonPath); UploadNewFileRecord uploadNewFileRecord = readInputJson(armResponseCreateRecord.getInput()); if (nonNull(uploadNewFileRecord)) { if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) { @@ -476,6 +506,16 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil } + private ArmResponseCreateRecord getResponseCreateRecordOrDelete(Path jsonPath) throws IOException { + try { + return objectMapper.readValue(jsonPath.toFile(), ArmResponseCreateRecord.class); + } catch (Exception e) { + log.error("Unable to read ARM response create record file {} - About to delete ", jsonPath.toFile().getAbsoluteFile(), e); + deleteResponseBlobs(List.of(jsonPath.toFile().getAbsolutePath())); + throw e; + } + } + private void processUploadResponseFiles(List uploadFileResponses, ArmBatchResponses armBatchResponses) { for (UploadFileFilenameProcessor uploadFileFilenameProcessor : uploadFileResponses) { try { @@ -503,7 +543,7 @@ private void readUploadFile(BinaryData uploadFileBinary, UploadFileFilenameProce if (jsonPath.toFile().exists()) { logResponseFileContents(jsonPath); - ArmResponseUploadFileRecord armResponseUploadFileRecord = objectMapper.readValue(jsonPath.toFile(), ArmResponseUploadFileRecord.class); + ArmResponseUploadFileRecord armResponseUploadFileRecord = getResponseUploadFileRecordOrDelete(jsonPath); UploadNewFileRecord uploadNewFileRecord = readInputJson(armResponseUploadFileRecord.getInput()); if (nonNull(uploadNewFileRecord)) { if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) { @@ -534,6 +574,16 @@ private void readUploadFile(BinaryData uploadFileBinary, UploadFileFilenameProce } } + private ArmResponseUploadFileRecord getResponseUploadFileRecordOrDelete(Path jsonPath) throws IOException { + try { + return objectMapper.readValue(jsonPath.toFile(), ArmResponseUploadFileRecord.class); + } catch (Exception e) { + log.error("Unable to read ARM response upload file {} - About to delete ", jsonPath.toFile().getAbsoluteFile(), e); + deleteResponseBlobs(List.of(jsonPath.toFile().getAbsolutePath())); + throw e; + } + } + private void logResponseFileContents(Path jsonPath) { try { String contents = FileUtils.readFileToString(jsonPath.toFile(), UTF_8); @@ -713,7 +763,7 @@ private void readInvalidLineFile(BinaryData invalidLineFileBinary, InvalidLineFi if (jsonPath.toFile().exists()) { logResponseFileContents(jsonPath); - ArmResponseInvalidLineRecord armResponseInvalidLineRecord = objectMapper.readValue(jsonPath.toFile(), ArmResponseInvalidLineRecord.class); + ArmResponseInvalidLineRecord armResponseInvalidLineRecord = getResponseInvalidLineRecordOrDelete(jsonPath); String input = armResponseInvalidLineRecord.getInput(); UploadNewFileRecord uploadNewFileRecord = readInputJson(input); if (nonNull(uploadNewFileRecord)) { @@ -748,6 +798,16 @@ private void readInvalidLineFile(BinaryData invalidLineFileBinary, InvalidLineFi } } + private ArmResponseInvalidLineRecord getResponseInvalidLineRecordOrDelete(Path jsonPath) throws IOException { + try { + return objectMapper.readValue(jsonPath.toFile(), ArmResponseInvalidLineRecord.class); + } catch (Exception e) { + log.error("Unable to read ARM response invalid line file {} - About to delete ", jsonPath.toFile().getAbsoluteFile(), e); + deleteResponseBlobs(List.of(jsonPath.toFile().getAbsolutePath())); + throw e; + } + } + private void processInvalidLineFile(int externalObjectDirectoryId, InvalidLineFileFilenameProcessor invalidLineFileFilenameProcessor, ArmResponseInvalidLineRecord armResponseInvalidLineRecord, CreateRecordFilenameProcessor createRecordFilenameProcessor,