Refactor MultiResourceItemWriter

spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/MultiResourceItemWriter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2006-2024 the original author or authors.
+ * Copyright 2006-2025 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -34,9 +34,6 @@
  * {@link #setItemCountLimitPerResource(int)}. Suffix creation can be customized with
  * {@link #setResourceSuffixCreator(ResourceSuffixCreator)}.
  * <p>
- * Note that new resources are created only at chunk boundaries i.e. the number of items
- * written into one resource is between the limit set by
- * <p>
  * This writer will create an output file only when there are items to write, which means
  * there would be no empty file created if no items are passed (for example when all items
  * are filtered or skipped during the processing phase).
@@ -45,6 +42,7 @@
  * @param <T> item type
  * @author Robert Kasanicky
  * @author Mahmoud Ben Hassine
+ * @author Henning Pöttker
  */
 public class MultiResourceItemWriter<T> extends AbstractItemStreamItemWriter<T> {
 
@@ -74,22 +72,30 @@ public MultiResourceItemWriter() {
 
 	@Override
 	public void write(Chunk<? extends T> items) throws Exception {
-		if (!opened) {
-			File file = setResourceToDelegate();
-			// create only if write is called
-			file.createNewFile();
-			Assert.state(file.canWrite(), "Output resource " + file.getAbsolutePath() + " must be writable");
-			delegate.open(new ExecutionContext());
-			opened = true;
-		}
-		delegate.write(items);
-		currentResourceItemCount += items.size();
-		if (currentResourceItemCount >= itemCountLimitPerResource) {
-			delegate.close();
-			resourceIndex++;
-			currentResourceItemCount = 0;
-			setResourceToDelegate();
-			opened = false;
+		int writtenItems = 0;
+		while (writtenItems < items.size()) {
+			if (!opened) {
+				File file = setResourceToDelegate();
+				// create only if write is called
+				file.createNewFile();
+				Assert.state(file.canWrite(), "Output resource " + file.getAbsolutePath() + " must be writable");
+				delegate.open(new ExecutionContext());
+				opened = true;
+			}
+
+			int itemsToWrite = Math.min(itemCountLimitPerResource - currentResourceItemCount,
+					items.size() - writtenItems);
+			delegate.write(new Chunk<T>(items.getItems().subList(writtenItems, writtenItems + itemsToWrite)));
+			currentResourceItemCount += itemsToWrite;
+			writtenItems += itemsToWrite;
+
+			if (currentResourceItemCount >= itemCountLimitPerResource) {
+				delegate.close();
+				resourceIndex++;
+				currentResourceItemCount = 0;
+				setResourceToDelegate();
+				opened = false;
+			}
 		}
 	}
 

spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/MultiResourceItemWriterFlatFileTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2008-2023 the original author or authors.
+ * Copyright 2008-2025 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -78,22 +78,22 @@ void testBasicMultiResourceWriteScenario() throws Exception {
 
 		tested.write(Chunk.of("1", "2", "3"));
 
-		File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
-		assertTrue(part1.exists());
-		assertEquals("123", readFile(part1));
+		assertFileExistsAndContains(1, "12");
+		assertFileExistsAndContains(2, "3");
 
 		tested.write(Chunk.of("4"));
-		File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
-		assertTrue(part2.exists());
-		assertEquals("4", readFile(part2));
+
+		assertFileExistsAndContains(2, "34");
 
 		tested.write(Chunk.of("5"));
-		assertEquals("45", readFile(part2));
+
+		assertFileExistsAndContains(3, "5");
 
 		tested.write(Chunk.of("6", "7", "8", "9"));
-		File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
-		assertTrue(part3.exists());
-		assertEquals("6789", readFile(part3));
+
+		assertFileExistsAndContains(3, "56");
+		assertFileExistsAndContains(4, "78");
+		assertFileExistsAndContains(5, "9");
 	}
 
 	@Test
@@ -107,7 +107,7 @@ void testUpdateAfterDelegateClose() throws Exception {
 		assertEquals(1, executionContext.getInt(tested.getExecutionContextKey("resource.index")));
 		tested.write(Chunk.of("1", "2", "3"));
 		tested.update(executionContext);
-		assertEquals(0, executionContext.getInt(tested.getExecutionContextKey("resource.item.count")));
+		assertEquals(1, executionContext.getInt(tested.getExecutionContextKey("resource.item.count")));
 		assertEquals(2, executionContext.getInt(tested.getExecutionContextKey("resource.index")));
 
 	}
@@ -121,17 +121,22 @@ void testMultiResourceWriteScenarioWithFooter() throws Exception {
 
 		tested.write(Chunk.of("1", "2", "3"));
 
-		File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
-		assertTrue(part1.exists());
+		assertFileExistsAndContains(1, "12f");
+		assertFileExistsAndContains(2, "3");
 
 		tested.write(Chunk.of("4"));
-		File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
-		assertTrue(part2.exists());
+
+		assertFileExistsAndContains(2, "34f");
+
+		tested.write(Chunk.of("5"));
+
+		assertFileExistsAndContains(3, "5");
 
 		tested.close();
 
-		assertEquals("123f", readFile(part1));
-		assertEquals("4f", readFile(part2));
+		assertFileExistsAndContains(1, "12f");
+		assertFileExistsAndContains(2, "34f");
+		assertFileExistsAndContains(3, "5f");
 
 	}
 
@@ -144,19 +149,18 @@ void testTransactionalMultiResourceWriteScenarioWithFooter() throws Exception {
 
 		ResourcelessTransactionManager transactionManager = new ResourcelessTransactionManager();
 
-		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2", "3")));
+		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2")));
 
-		File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
-		assertTrue(part1.exists());
+		assertFileExistsAndContains(1, "12f");
 
-		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("4")));
-		File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
-		assertTrue(part2.exists());
+		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("3")));
+
+		assertFileExistsAndContains(2, "3");
 
 		tested.close();
 
-		assertEquals("123f", readFile(part1));
-		assertEquals("4f", readFile(part2));
+		assertFileExistsAndContains(1, "12f");
+		assertFileExistsAndContains(2, "3f");
 
 	}
 
@@ -168,27 +172,23 @@ void testRestart() throws Exception {
 
 		tested.write(Chunk.of("1", "2", "3"));
 
-		File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
-		assertTrue(part1.exists());
-		assertEquals("123", readFile(part1));
-
-		tested.write(Chunk.of("4"));
-		File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
-		assertTrue(part2.exists());
-		assertEquals("4", readFile(part2));
+		assertFileExistsAndContains(1, "12");
+		assertFileExistsAndContains(2, "3");
 
 		tested.update(executionContext);
 		tested.close();
 
 		tested.open(executionContext);
 
-		tested.write(Chunk.of("5"));
-		assertEquals("45", readFile(part2));
+		tested.write(Chunk.of("4"));
 
-		tested.write(Chunk.of("6", "7", "8", "9"));
-		File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
-		assertTrue(part3.exists());
-		assertEquals("6789", readFile(part3));
+		assertFileExistsAndContains(2, "34");
+
+		tested.write(Chunk.of("5", "6", "7", "8", "9"));
+
+		assertFileExistsAndContains(3, "56");
+		assertFileExistsAndContains(4, "78");
+		assertFileExistsAndContains(5, "9");
 	}
 
 	@Test
@@ -201,27 +201,24 @@ void testRestartWithFooter() throws Exception {
 
 		tested.write(Chunk.of("1", "2", "3"));
 
-		File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
-		assertTrue(part1.exists());
-		assertEquals("123f", readFile(part1));
-
-		tested.write(Chunk.of("4"));
-		File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
-		assertTrue(part2.exists());
-		assertEquals("4", readFile(part2));
+		assertFileExistsAndContains(1, "12f");
+		assertFileExistsAndContains(2, "3");
 
 		tested.update(executionContext);
 		tested.close();
 
 		tested.open(executionContext);
 
-		tested.write(Chunk.of("5"));
-		assertEquals("45f", readFile(part2));
+		tested.write(Chunk.of("4"));
 
-		tested.write(Chunk.of("6", "7", "8", "9"));
-		File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
-		assertTrue(part3.exists());
-		assertEquals("6789f", readFile(part3));
+		assertFileExistsAndContains(2, "34f");
+
+		tested.write(Chunk.of("5", "6", "7", "8", "9"));
+		tested.close();
+
+		assertFileExistsAndContains(3, "56f");
+		assertFileExistsAndContains(4, "78f");
+		assertFileExistsAndContains(5, "9f");
 	}
 
 	@Test
@@ -233,24 +230,28 @@ void testTransactionalRestartWithFooter() throws Exception {
 
 		ResourcelessTransactionManager transactionManager = new ResourcelessTransactionManager();
 
-		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2", "3")));
+		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2")));
 
-		File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
-		assertTrue(part1.exists());
-		assertEquals("123f", readFile(part1));
+		assertFileExistsAndContains(1, "12f");
 
-		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("4")));
-		File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
-		assertTrue(part2.exists());
-		assertEquals("4", readFile(part2));
+		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("3")));
+
+		assertFileExistsAndContains(2, "3");
 
 		tested.update(executionContext);
 		tested.close();
 
 		tested.open(executionContext);
 
-		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("5")));
-		assertEquals("45f", readFile(part2));
+		new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("4")));
+
+		assertFileExistsAndContains(2, "34f");
+	}
+
+	private void assertFileExistsAndContains(int index, String expected) throws Exception {
+		File part = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(index));
+		assertTrue(part.exists());
+		assertEquals(expected, readFile(part));
 	}
 
 }

spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/MultiResourceItemWriterXmlTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2022 the original author or authors.
+ * Copyright 2009-2025 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -108,21 +108,26 @@ void multiResourceWritingWithRestart() throws Exception {
 		tested.update(executionContext);
 		tested.close();
 
-		assertEquals(xmlDocStart + "<prefix:4/>" + xmlDocEnd, readFile(part2));
-		assertEquals(xmlDocStart + "<prefix:1/><prefix:2/><prefix:3/>" + xmlDocEnd, readFile(part1));
+		assertEquals(xmlDocStart + "<prefix:3/><prefix:4/>" + xmlDocEnd, readFile(part2));
+		assertEquals(xmlDocStart + "<prefix:1/><prefix:2/>" + xmlDocEnd, readFile(part1));
 
 		tested.open(executionContext);
 
 		tested.write(Chunk.of("5"));
-
-		tested.write(Chunk.of("6", "7", "8", "9"));
 		File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
 		assertTrue(part3.exists());
 
+		tested.write(Chunk.of("6", "7", "8", "9"));
+		File part4 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(4));
+		assertTrue(part4.exists());
+		File part5 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(5));
+		assertTrue(part5.exists());
+
 		tested.close();
 
-		assertEquals(xmlDocStart + "<prefix:4/><prefix:5/>" + xmlDocEnd, readFile(part2));
-		assertEquals(xmlDocStart + "<prefix:6/><prefix:7/><prefix:8/><prefix:9/>" + xmlDocEnd, readFile(part3));
+		assertEquals(xmlDocStart + "<prefix:5/><prefix:6/>" + xmlDocEnd, readFile(part3));
+		assertEquals(xmlDocStart + "<prefix:7/><prefix:8/>" + xmlDocEnd, readFile(part4));
+		assertEquals(xmlDocStart + "<prefix:9/>" + xmlDocEnd, readFile(part5));
 	}
 
 }