NIFI-13996 Improved removal of temporary files after a build (apache#…

…9551) Signed-off-by: David Handerman <exceptionfactory@apache.org> Co-authored-by: David Handermann <exceptionfactory@apache.org>
jrsteinebrey · Dec 2, 2024 · 82fe051 · 82fe051
1 parent 1c9f083
commit 82fe051
Show file tree

Hide file tree

Showing 15 changed files with 221 additions and 70 deletions.
diff --git a/...t/java/org/apache/nifi/c2/client/service/operation/TransferDebugOperationHandlerTest.java b/...t/java/org/apache/nifi/c2/client/service/operation/TransferDebugOperationHandlerTest.java
@@ -18,6 +18,7 @@
 package org.apache.nifi.c2.client.service.operation;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.newDirectoryStream;
 import static java.nio.file.Files.write;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
@@ -46,6 +47,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.HashMap;
@@ -56,9 +58,11 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.io.FileUtils;
 import org.apache.nifi.c2.client.api.C2Client;
 import org.apache.nifi.c2.protocol.api.C2Operation;
 import org.apache.nifi.c2.protocol.api.C2OperationAck;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.io.TempDir;
@@ -87,6 +91,18 @@ public class TransferDebugOperationHandlerTest {
     @TempDir
     private File tempDir;
 
+    @AfterEach
+    public void cleanUpAfterEach() {
+        // Cleanup all the temporary operationId[0-9]* directories which are generated by TransferDebugOperationHandler.
+        final Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"));
+        try (DirectoryStream<Path> directoryStream = newDirectoryStream(tempDir, "operationId[0-9]*")) {
+            for (Path operationId : directoryStream) {
+                FileUtils.deleteDirectory(operationId.toFile());
+            }
+        } catch (Exception ignore) {
+        }
+    }
+
     private static Stream<Arguments> invalidConstructorArguments() {
         C2Client mockC2Client = mock(C2Client.class);
         return Stream.of(

diff --git a/.../java/org/apache/nifi/minifi/c2/command/syncresource/DefaultSyncResourceStrategyTest.java b/.../java/org/apache/nifi/minifi/c2/command/syncresource/DefaultSyncResourceStrategyTest.java
@@ -19,6 +19,7 @@
 
 import static java.lang.Boolean.TRUE;
 import static java.nio.file.Files.createTempFile;
+import static java.nio.file.Files.newDirectoryStream;
 import static java.util.Optional.empty;
 import static java.util.Optional.ofNullable;
 import static org.apache.nifi.c2.protocol.api.C2OperationState.OperationState.FULLY_APPLIED;
@@ -38,17 +39,21 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Stream;
+
 import org.apache.nifi.c2.protocol.api.C2OperationState.OperationState;
 import org.apache.nifi.c2.protocol.api.ResourceItem;
 import org.apache.nifi.c2.protocol.api.ResourceType;
 import org.apache.nifi.c2.protocol.api.ResourcesGlobalHash;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -77,6 +82,18 @@ public class DefaultSyncResourceStrategyTest {
 
     private DefaultSyncResourceStrategy testSyncResourceStrategy;
 
+    @AfterAll
+    public static void cleanUpAfterAll() {
+        // Cleanup all the temporary uuid-<large numbers>.tmp files which are generated by DefaultSyncResourceStrategy.
+        final Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"));
+        try (DirectoryStream<Path> directoryStream = newDirectoryStream(tempDir, "[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*[0-9]*.tmp")) {
+            for (Path tmpFile : directoryStream) {
+               Files.deleteIfExists(tmpFile);
+            }
+        } catch (Exception ignore) {
+        }
+    }
+
     @BeforeEach
     public void setup() {
         testSyncResourceStrategy = new DefaultSyncResourceStrategy(mockResourceRepository);

diff --git a/...utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcClobReadable.java b/...utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcClobReadable.java
@@ -25,13 +25,13 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.Files;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
@@ -46,10 +46,12 @@ public class TestJdbcClobReadable {
 
     private static final String DERBY_LOG_PROPERTY = "derby.stream.error.file";
 
+    @TempDir
+    private static File tempDir;
+
     @BeforeAll
     public static void setDerbyLog() {
-        final File derbyLog = new File(System.getProperty("java.io.tmpdir"), "derby.log");
-        derbyLog.deleteOnExit();
+        final File derbyLog = new File(tempDir, "derby.log");
         System.setProperty(DERBY_LOG_PROPERTY, derbyLog.getAbsolutePath());
     }
 
@@ -101,9 +103,8 @@ public void testDriverLoad() throws ClassNotFoundException, SQLException {
     private File folder;
 
     private void validateClob(String someClob) throws SQLException, ClassNotFoundException, IOException {
-        folder = Files.createTempDirectory(String.valueOf(System.currentTimeMillis()))
-                .resolve("db")
-                .toFile();
+        File topLevelTempDir = new File(tempDir, String.valueOf(System.currentTimeMillis()));
+        folder = new File(topLevelTempDir, "db");
         final Connection con = createConnection(folder.getAbsolutePath());
         final Statement st = con.createStatement();
 

diff --git a/...n-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcHugeStream.java b/...n-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcHugeStream.java
@@ -21,11 +21,11 @@
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.io.DatumReader;
 import org.apache.derby.jdbc.EmbeddedDriver;
-import org.apache.nifi.util.file.FileUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -57,12 +57,15 @@
  *
  */
 public class TestJdbcHugeStream {
+
     private static final String DERBY_LOG_PROPERTY = "derby.stream.error.file";
 
+    @TempDir
+    private static File tempDir;
+
     @BeforeAll
     public static void setDerbyLog() {
-        final File derbyLog = new File(System.getProperty("java.io.tmpdir"), "derby.log");
-        derbyLog.deleteOnExit();
+        final File derbyLog = new File(tempDir, "derby.log");
         System.setProperty(DERBY_LOG_PROPERTY, derbyLog.getAbsolutePath());
     }
 
@@ -71,17 +74,16 @@ public static void setDerbyLog() {
     @BeforeEach
     public void setup() throws IOException, SQLException {
         DriverManager.registerDriver(new EmbeddedDriver());
-        tempFile = Files.createTempDirectory(String.valueOf(System.currentTimeMillis()))
-                .resolve("db")
-                .toFile();
+        File topLevelTempDir = new File(tempDir, String.valueOf(System.currentTimeMillis()));
+        Files.createDirectories(topLevelTempDir.toPath());
+        tempFile = new File(topLevelTempDir, "db");
     }
 
     @AfterEach
-    public void cleanup() throws IOException {
+    public void cleanup() {
         if (tempFile != null && tempFile.exists()) {
             final SQLException exception = assertThrows(SQLException.class, () -> DriverManager.getConnection("jdbc:derby:;shutdown=true"));
             assertEquals("XJ015", exception.getSQLState());
-            FileUtils.deleteFile(tempFile, true);
         }
     }
 
@@ -137,7 +139,7 @@ record = dataFileReader.next(record);
     static String createProducts = "create table products (id integer, name varchar(100), code integer)";
     static String createRelationships = "create table relationships (id integer,name varchar(100), code integer)";
 
-    static public void loadTestData2Database(Connection con, int nrOfPersons, int nrOfProducts, int nrOfRels) throws ClassNotFoundException, SQLException {
+    static public void loadTestData2Database(Connection con, int nrOfPersons, int nrOfProducts, int nrOfRels) throws SQLException {
 
         final Statement st = con.createStatement();
 

diff --git a/...sion-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcTypesH2.java b/...sion-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcTypesH2.java
@@ -16,16 +16,14 @@
  */
 package org.apache.nifi.util.db;
 
-import org.apache.nifi.util.file.FileUtils;
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 
 import java.io.ByteArrayOutputStream;
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
+import java.nio.file.Path;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
@@ -64,19 +62,16 @@ public class TestJdbcTypesH2 {
 
     String dbPath;
 
+    @TempDir
+    private Path tempDir;
+
     @BeforeEach
     public void beforeEach() throws IOException {
-        dbPath = Files.createTempDirectory(String.valueOf(System.currentTimeMillis()))
-                .resolve("db")
+        dbPath = tempDir.resolve("db")
                 .toFile()
                 .getAbsolutePath();
     }
 
-    @AfterEach
-    public void afterEach() throws IOException {
-        FileUtils.deleteFile(new File(dbPath), true);
-    }
-
     @Test
     public void testSQLTypesMapping() throws SQLException, IOException {
         final Connection con = createConnection(dbPath);

diff --git a/...-processors/src/test/java/org/apache/nifi/processors/groovyx/ExecuteGroovyScriptTest.java b/...-processors/src/test/java/org/apache/nifi/processors/groovyx/ExecuteGroovyScriptTest.java
@@ -39,13 +39,15 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.DisabledOnOs;
 import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.io.TempDir;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.PrintStream;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -55,6 +57,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -163,11 +166,12 @@ public void testReadFlowFileContentAndStoreInFlowFileAttribute() throws Exceptio
     }
 
     @Test
-    public void testAdditionalClasspath() throws Exception {
+    public void testAdditionalClasspath(@TempDir File tempDir) throws Exception {
         Set<URL> expectedClasspathURLs = new HashSet<>();
         StringBuilder additionalClasspath = new StringBuilder();
         for (int i = 0; i < 3; i++) {
-            Path p = java.nio.file.Files.createTempFile(getClass().getName(), ".tmp");
+            Path p = new File(tempDir, getClass().getName() + UUID.randomUUID() + ".tmp").toPath();
+            Files.createFile(p);
             expectedClasspathURLs.add(p.toUri().toURL());
             additionalClasspath.append(p);
             additionalClasspath.append(i == 0 ? ',' : ';'); // create additional classpath string separated by ; and ,

diff --git a/...dle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java b/...dle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java
@@ -43,13 +43,13 @@
 import org.ietf.jgss.GSSException;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.mockito.Mockito;
 
 import javax.security.sasl.SaslException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.Collection;
@@ -710,7 +710,7 @@ public void testPutFileWithCloseException() throws IOException {
     }
 
     @Test
-    public void testPutFileFromLocalFile() throws Exception {
+    public void testPutFileFromLocalFile(@TempDir java.nio.file.Path tempDir) throws Exception {
         final FileSystem spyFileSystem = Mockito.spy(mockFileSystem);
         final PutHDFS proc = new TestablePutHDFS(spyFileSystem);
         final TestRunner runner = TestRunners.newTestRunner(proc);
@@ -723,16 +723,15 @@ public void testPutFileFromLocalFile() throws Exception {
 
         String serviceId = FileResourceService.class.getSimpleName();
         FileResourceService service = new StandardFileResourceService();
-        byte[] FILE_DATA = "0123456789".getBytes(StandardCharsets.UTF_8);
         byte[] EMPTY_CONTENT = new byte[0];
         runner.addControllerService(serviceId, service);
         runner.setProperty(service, StandardFileResourceService.FILE_PATH, String.format("${%s}", attributeName));
         runner.enableControllerService(service);
 
         runner.setProperty(ResourceTransferProperties.RESOURCE_TRANSFER_SOURCE, ResourceTransferSource.FILE_RESOURCE_SERVICE.getValue());
         runner.setProperty(ResourceTransferProperties.FILE_RESOURCE_SERVICE, serviceId);
-        java.nio.file.Path tempFilePath = Files.createTempFile("PutHDFS_testPutFileFromLocalFile_", "");
-        Files.write(tempFilePath, FILE_DATA);
+        java.nio.file.Path tempFilePath = tempDir.resolve("PutHDFS_testPutFileFromLocalFile_" + System.currentTimeMillis());
+        Files.writeString(tempFilePath, "0123456789");
 
         Map<String, String> attributes = new HashMap<>();
 

diff --git a/.../nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelHeaderSchemaStrategy.java b/.../nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelHeaderSchemaStrategy.java
@@ -28,6 +28,7 @@
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.xssf.usermodel.XSSFSheet;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
@@ -37,9 +38,13 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
 
+import static java.nio.file.Files.newDirectoryStream;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -52,6 +57,21 @@ public class TestExcelHeaderSchemaStrategy {
     @Mock
     ComponentLog logger;
 
+    /*
+     * Cleanup the temporary poifiles directory which is created by org.apache.poi.util.DefaultTempFileCreationStrategy
+     * the strategy org.apache.poi.util.TempFile uses which in turn is used by com.github.pjfanning.xlsx.impl.StreamingSheetReader.
+     */
+    @AfterAll
+    public static void cleanUpAfterAll() {
+        final Path tempDir = Path.of(System.getProperty("java.io.tmpdir")).resolve("poifiles");
+        try (DirectoryStream<Path> directoryStream = newDirectoryStream(tempDir, "tmp-[0-9]*.xlsx")) {
+            for (Path tmpFile : directoryStream) {
+                Files.deleteIfExists(tmpFile);
+            }
+        } catch (Exception ignore) {
+        }
+    }
+
     @Test
     void testWhereConfiguredStartRowIsEmpty() throws IOException {
         Object[][] data = {{}, {1, "Manny"}, {2, "Moe"}, {3, "Jack"}};