From 65f7fe81365eb159f3ab7f66308aadb25ed8fb4a Mon Sep 17 00:00:00 2001 From: John Scancella Date: Fri, 12 Jan 2018 11:54:29 -0500 Subject: [PATCH] refs #99 - don't write payload file to data directory if that file is in the fetch list --- .../bagit/writer/PayloadWriter.java | 41 ++++++++++++++----- src/main/resources/MessageBundle.properties | 3 +- .../bagit/writer/PayloadWriterTest.java | 21 +++++++++- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java b/src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java index 4e3b1df25..2760d5605 100644 --- a/src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java +++ b/src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java @@ -4,6 +4,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.HashSet; +import java.util.List; import java.util.ResourceBundle; import java.util.Set; @@ -11,6 +13,7 @@ import org.slf4j.LoggerFactory; import gov.loc.repository.bagit.domain.Bag; +import gov.loc.repository.bagit.domain.FetchItem; import gov.loc.repository.bagit.domain.Manifest; import gov.loc.repository.bagit.domain.Version; @@ -35,12 +38,12 @@ static Path writeVersionDependentPayloadFiles(final Bag bag, final Path outputDi if(bag.getVersion().isSameOrNewer(VERSION_2_0)){ bagitDir = outputDir.resolve(".bagit"); Files.createDirectories(bagitDir); - writePayloadFiles(bag.getPayLoadManifests(), outputDir, bag.getRootDir()); + writePayloadFiles(bag.getPayLoadManifests(), bag.getItemsToFetch(), outputDir, bag.getRootDir()); } else{ final Path dataDir = outputDir.resolve("data"); Files.createDirectories(dataDir); - writePayloadFiles(bag.getPayLoadManifests(), dataDir, bag.getRootDir().resolve("data")); + writePayloadFiles(bag.getPayLoadManifests(), bag.getItemsToFetch(), dataDir, bag.getRootDir().resolve("data")); } return bagitDir; @@ -50,25 +53,41 @@ static Path writeVersionDependentPayloadFiles(final Bag bag, final Path outputDi * Write the payload file(s) to the output directory * * @param payloadManifests the set of objects representing the payload manifests + * @param fetchItems the list of items to exclude from writing in the output directory because they will be fetched * @param outputDir the data directory of the bag * @param bagDataDir the data directory of the bag * * @throws IOException if there was a problem writing a file */ - public static void writePayloadFiles(final Set payloadManifests, final Path outputDir, final Path bagDataDir) throws IOException{ + public static void writePayloadFiles(final Set payloadManifests, final List fetchItems, final Path outputDir, final Path bagDataDir) throws IOException{ logger.info(messages.getString("writing_payload_files")); + final Set fetchPaths = getFetchPaths(fetchItems); + for(final Manifest payloadManifest : payloadManifests){ for(final Path payloadFile : payloadManifest.getFileToChecksumMap().keySet()){ - final Path relativePayloadPath = bagDataDir.relativize(payloadFile); - - final Path writeToPath = outputDir.resolve(relativePayloadPath); - logger.debug(messages.getString("writing_payload_file_to_path"), payloadFile, writeToPath); - final Path parent = writeToPath.getParent(); - if(parent != null){ - Files.createDirectories(parent); + final Path relativePayloadPath = bagDataDir.relativize(payloadFile); + + if(fetchPaths.contains(relativePayloadPath.normalize())) { + logger.info(messages.getString("skip_fetch_item_when_writing_payload"), payloadFile); + } + else { + final Path writeToPath = outputDir.resolve(relativePayloadPath); + logger.debug(messages.getString("writing_payload_file_to_path"), payloadFile, writeToPath); + final Path parent = writeToPath.getParent(); + if(parent != null){ + Files.createDirectories(parent); + } + Files.copy(payloadFile, writeToPath, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); } - Files.copy(payloadFile, writeToPath, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); } } } + + private static Set getFetchPaths(final List fetchItems){ + final Set fetchPaths = new HashSet<>(); + for(final FetchItem fetchItem : fetchItems) { + fetchPaths.add(fetchItem.getPath()); + } + return fetchPaths; + } } diff --git a/src/main/resources/MessageBundle.properties b/src/main/resources/MessageBundle.properties index cf372dcfe..33e749e3f 100644 --- a/src/main/resources/MessageBundle.properties +++ b/src/main/resources/MessageBundle.properties @@ -217,4 +217,5 @@ writing_manifest_to_path=Writing manifest to [{}]. writing_metadata_to_path=Writing bag metadata file [{}] to [{}]. #for PayloadWriter.java -writing_payload_file_to_path=Writing payload file [{}] to [{}]. \ No newline at end of file +writing_payload_file_to_path=Writing payload file [{}] to [{}]. +skip_fetch_item_when_writing_payload=Skipping payload file {} because it is in the fetch list. \ No newline at end of file diff --git a/src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java b/src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java index c42484530..5e907597b 100644 --- a/src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java +++ b/src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java @@ -6,6 +6,8 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -14,6 +16,7 @@ import org.junit.rules.TemporaryFolder; import gov.loc.repository.bagit.PrivateConstructorTest; +import gov.loc.repository.bagit.domain.FetchItem; import gov.loc.repository.bagit.domain.Manifest; import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; @@ -39,7 +42,23 @@ public void testWritePayloadFiles() throws IOException, URISyntaxException{ File copiedFile = new File(outputDir, "data/dir1/test3.txt"); assertFalse(copiedFile.exists() || copiedFile.getParentFile().exists()); - PayloadWriter.writePayloadFiles(payloadManifests, Paths.get(outputDir.toURI()), rootDir); + PayloadWriter.writePayloadFiles(payloadManifests, new ArrayList<>(),Paths.get(outputDir.toURI()), rootDir); assertTrue(copiedFile.exists() && copiedFile.getParentFile().exists()); } + + @Test + public void testWritePayloadFilesMinusFetchFiles() throws IOException, URISyntaxException{ + Path rootDir = Paths.get(getClass().getClassLoader().getResource("bags/v0_97/bag").toURI()); + Path testFile = Paths.get(getClass().getClassLoader().getResource("bags/v0_97/bag/data/dir1/test3.txt").toURI()); + Manifest manifest = new Manifest(StandardSupportedAlgorithms.MD5); + manifest.getFileToChecksumMap().put(testFile, "someHashValue"); + Set payloadManifests = new HashSet<>(); + payloadManifests.add(manifest); + File outputDir = folder.newFolder(); + File copiedFile = new File(outputDir, "data/dir1/test3.txt"); + + assertFalse(copiedFile.exists() || copiedFile.getParentFile().exists()); + PayloadWriter.writePayloadFiles(payloadManifests, Arrays.asList(new FetchItem(null, null, Paths.get("data/dir1/test3.txt"))),Paths.get(outputDir.toURI()), rootDir); + assertFalse(copiedFile.exists() && copiedFile.getParentFile().exists()); + } }