From 344255f9e1338fcbe064f8a26bfe22c36a953ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20P=C3=B6hler?= Date: Fri, 22 Oct 2021 14:20:36 +0200 Subject: [PATCH] TempFileProvider base code WeakRef: Delete file --- .../java/io/ebean/config/DatabaseConfig.java | 10 + .../DeleteOnShutdownTempFileProvider.java | 68 +++++++ .../io/ebean/config/TempFileProvider.java | 23 +++ .../ebean/config/WeakRefTempFileProvider.java | 145 +++++++++++++++ .../io/ebean/TestWeakRefTempFileProvider.java | 171 ++++++++++++++++++ .../server/core/DefaultServer.java | 3 + .../server/core/InternalConfiguration.java | 7 + .../server/type/DefaultTypeManager.java | 3 +- .../server/type/ScalarTypeFile.java | 21 +-- 9 files changed, 438 insertions(+), 13 deletions(-) create mode 100644 ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java create mode 100644 ebean-api/src/main/java/io/ebean/config/TempFileProvider.java create mode 100644 ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java create mode 100644 ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java diff --git a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java index 8d614f803c..a3277f9fda 100644 --- a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java +++ b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java @@ -398,6 +398,8 @@ public class DatabaseConfig { */ private Clock clock = Clock.systemUTC(); + private TempFileProvider tempFileProvider = new WeakRefTempFileProvider(); + private List idGenerators = new ArrayList<>(); private List findControllers = new ArrayList<>(); private List persistControllers = new ArrayList<>(); @@ -569,6 +571,14 @@ public void setClock(final Clock clock) { this.clock = clock; } + public TempFileProvider getTempFileProvider() { + return tempFileProvider; + } + + public void setTempFileProvider(final TempFileProvider tempFileProvider) { + this.tempFileProvider = tempFileProvider; + } + /** * Return the slow query time in millis. */ diff --git a/ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java b/ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java new file mode 100644 index 0000000000..ad257c98ee --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java @@ -0,0 +1,68 @@ +package io.ebean.config; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * TempFileProvider implementation, which deletes all temp files on shutdown. + * + * @author Roland Praml, FOCONIS AG + * + */ +public class DeleteOnShutdownTempFileProvider implements TempFileProvider { + + private static final Logger logger = LoggerFactory.getLogger(DeleteOnShutdownTempFileProvider.class); + + List tempFiles = new ArrayList<>(); + private final String prefix; + private final String suffix; + private final File directory; + + /** + * Creates the TempFileProvider with default prefix "db-". + */ + public DeleteOnShutdownTempFileProvider() { + this("db-", null, null); + } + + /** + * Creates the TempFileProvider. + */ + public DeleteOnShutdownTempFileProvider(String prefix, String suffix, File directory) { + this.prefix = prefix; + this.suffix = suffix; + this.directory = directory; + } + + @Override + public File createTempFile() throws IOException { + File file = File.createTempFile(prefix, suffix, directory); + synchronized (tempFiles) { + tempFiles.add(file.getAbsolutePath()); + } + return file; + } + + /** + * Deletes all created files on shutdown. + */ + @Override + public void shutdown() { + synchronized (tempFiles) { + for (String path : tempFiles) { + if (new File(path).delete()) { + logger.trace("deleted {}", path); + } else { + logger.warn("could not delete {}", path); + } + } + tempFiles.clear(); + } + } + +} diff --git a/ebean-api/src/main/java/io/ebean/config/TempFileProvider.java b/ebean-api/src/main/java/io/ebean/config/TempFileProvider.java new file mode 100644 index 0000000000..4658b46c28 --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/TempFileProvider.java @@ -0,0 +1,23 @@ +package io.ebean.config; + +import java.io.File; +import java.io.IOException; + +/** + * Creates a temp file for the ScalarTypeFile datatype. + * + * @author Roland Praml, FOCONIS AG + * + */ +public interface TempFileProvider { + + /** + * Creates a tempFile. + */ + File createTempFile() throws IOException; + + /** + * Shutdown the tempFileProvider. + */ + void shutdown(); +} diff --git a/ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java b/ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java new file mode 100644 index 0000000000..aa3ae82905 --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java @@ -0,0 +1,145 @@ +package io.ebean.config; + +import java.io.File; +import java.io.IOException; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * WeakRefTempFileProvider will delete the tempFile if all references to the returned File + * object are collected by the garbage collection. + * + * @author Roland Praml, FOCONIS AG + * + */ +public class WeakRefTempFileProvider implements TempFileProvider { + + private static final Logger logger = LoggerFactory.getLogger(WeakRefTempFileProvider.class); + + private final ReferenceQueue tempFiles = new ReferenceQueue<>(); + + private WeakFileReference root; + + private final String prefix; + private final String suffix; + private final File directory; + + /** + * We hold a linkedList of weak references. So we can remove stale files in O(1) + * + * @author Roland Praml, FOCONIS AG + */ + private static class WeakFileReference extends WeakReference { + + String path; + WeakFileReference prev; + WeakFileReference next; + + WeakFileReference(File referent, ReferenceQueue q) { + super(referent, q); + path = referent.getAbsolutePath(); + } + + boolean delete(boolean shutdown) { + File file = new File(path); + if (!file.exists()) { + logger.trace("already deleted {}", path); + return true; + } else if (file.delete()) { + logger.trace("deleted {}", path); + return true; + } else { + if (shutdown) { + logger.warn("could not delete {}", path); + } else { + logger.info("could not delete {} - will delete on shutdown", path); + } + return false; + } + } + } + + + /** + * Creates the TempFileProvider with default prefix "db-". + */ + public WeakRefTempFileProvider() { + this("db-", null, null); + } + + /** + * Creates the TempFileProvider. + */ + public WeakRefTempFileProvider(String prefix, String suffix, File directory) { + this.prefix = prefix; + this.suffix = suffix; + this.directory = directory; + } + + @Override + public File createTempFile() throws IOException { + File tempFile = File.createTempFile(prefix, suffix, directory); + logger.trace("createTempFile: {}", tempFile); + synchronized (this) { + add(new WeakFileReference(tempFile, tempFiles)); + } + return tempFile; + } + + /** + * Will delete stale files. + * This is public to use in tests. + */ + public void deleteStaleTempFiles() { + synchronized (this) { + deleteStaleTempFilesInternal(); + } + } + + private void deleteStaleTempFilesInternal() { + WeakFileReference ref; + while ((ref = (WeakFileReference) tempFiles.poll()) != null) { + if (ref.delete(false)) { + remove(ref); // remove from linkedList only, if delete was successful. + } + } + } + + private void add(WeakFileReference ref) { + deleteStaleTempFilesInternal(); + + if (root == null) { + root = ref; + } else { + ref.next = root; + root.prev = ref; + root = ref; + } + } + + private void remove(WeakFileReference ref) { + if (ref.next != null) { + ref.next.prev = ref.prev; + } + if (ref.prev != null) { + ref.prev.next = ref.next; + } else { + root = ref.next; + } + } + + /** + * Deletes all created files on shutdown. + */ + @Override + public void shutdown() { + while (root != null) { + root.delete(true); + root = root.next; + } + } + +} diff --git a/ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java b/ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java new file mode 100644 index 0000000000..dc69cc6876 --- /dev/null +++ b/ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java @@ -0,0 +1,171 @@ +package io.ebean; + + +import io.ebean.config.WeakRefTempFileProvider; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.channels.FileLock; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Test for the WeakRefTempFileProvider. (Note: this test relies on an aggressive garbage collection. + * if GC implementation will change, the test may fail) + * + * @author Roland Praml, FOCONIS AG + */ +public class TestWeakRefTempFileProvider { + + WeakRefTempFileProvider prov = new WeakRefTempFileProvider(); + + @AfterEach + public void shutdown() { + prov.shutdown(); + } + + /** + * Run the garbage collection and delete stale files. + */ + private void gc() throws InterruptedException { + System.gc(); + Thread.sleep(100); + prov.deleteStaleTempFiles(); + } + + @Test + public void testStaleEntries() throws Exception { + File tempFile = prov.createTempFile(); + String fileName = tempFile.getAbsolutePath(); + + gc(); + + assertThat(new File(fileName)).exists(); + + tempFile = null; // give up reference + + gc(); + + assertThat(new File(fileName)).doesNotExist(); + + + } + + @Test + public void testLinkedListForward() throws Exception { + File tempFile1 = prov.createTempFile(); + String fileName1 = tempFile1.getAbsolutePath(); + File tempFile2 = prov.createTempFile(); + String fileName2 = tempFile2.getAbsolutePath(); + File tempFile3 = prov.createTempFile(); + String fileName3 = tempFile3.getAbsolutePath(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).exists(); + + gc(); + + // give up first ref + tempFile1 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).exists(); + + // give up second ref + tempFile2 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).exists(); + + // give up third ref + tempFile3 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).doesNotExist(); + + } + + + @Test + public void testLinkedListReverse() throws Exception { + File tempFile1 = prov.createTempFile(); + String fileName1 = tempFile1.getAbsolutePath(); + File tempFile2 = prov.createTempFile(); + String fileName2 = tempFile2.getAbsolutePath(); + File tempFile3 = prov.createTempFile(); + String fileName3 = tempFile3.getAbsolutePath(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).exists(); + + gc(); + + // give up third ref + tempFile3 = null; + + gc(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).doesNotExist(); + + // give up second ref + tempFile2 = null; + + gc(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).doesNotExist(); + + // give up first ref + tempFile1 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).doesNotExist(); + + } + + @Test + @Disabled("Runs on Windows only") + public void testFileLocked() throws Exception { + File tempFile = prov.createTempFile(); + String fileName = tempFile.getAbsolutePath(); + + try (FileOutputStream os = new FileOutputStream(fileName)) { + FileLock lock = os.getChannel().lock(); + try { + os.write(42); + + tempFile = null; + gc(); + } finally { + lock.release(); + } + + } + + assertThat(new File(fileName)).exists(); + + prov.shutdown(); + + assertThat(new File(fileName)).doesNotExist(); + } +} diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java b/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java index 98356f4186..7d55b2b940 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java @@ -79,6 +79,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { private final String serverName; private final DatabasePlatform databasePlatform; private final TransactionManager transactionManager; + private final TempFileProvider tempFileProvider; private final QueryPlanManager queryPlanManager; private final ExtraMetrics extraMetrics; private final DataTimeZone dataTimeZone; @@ -158,6 +159,7 @@ public DefaultServer(InternalConfiguration config, ServerCacheManager cache) { this.queryPlanManager = config.initQueryPlanManager(transactionManager); this.metaInfoManager = new DefaultMetaInfoManager(this, this.config.getMetricNaming()); this.serverPlugins = config.getPlugins(); + this.tempFileProvider = config.getConfig().getTempFileProvider(); this.ddlGenerator = config.initDdlGenerator(this); this.scriptRunner = new DScriptRunner(this); @@ -373,6 +375,7 @@ public void shutdown(boolean shutdownDataSource, boolean deregisterDriver) { backgroundExecutor.shutdown(); // shutdown DataSource (if its an Ebean one) transactionManager.shutdown(shutdownDataSource, deregisterDriver); + tempFileProvider.shutdown(); dumpMetrics(); shutdown = true; if (shutdownDataSource) { diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java b/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java index 080d95560c..3ea3a186b8 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java @@ -8,6 +8,7 @@ import io.ebean.config.ExternalTransactionManager; import io.ebean.config.ProfilingConfig; import io.ebean.config.SlowQueryListener; +import io.ebean.config.TempFileProvider; import io.ebean.config.dbplatform.DatabasePlatform; import io.ebean.config.dbplatform.DbHistorySupport; import io.ebean.event.changelog.ChangeLogListener; @@ -76,6 +77,7 @@ public final class InternalConfiguration { private final DatabasePlatform databasePlatform; private final DeployInherit deployInherit; private final TypeManager typeManager; + private final TempFileProvider tempFileProvider; private final DtoBeanManager dtoBeanManager; private final ClockService clockService; private final DataTimeZone dataTimeZone; @@ -116,6 +118,7 @@ public final class InternalConfiguration { this.databasePlatform = config.getDatabasePlatform(); this.expressionFactory = initExpressionFactory(config); this.typeManager = new DefaultTypeManager(config, bootupClasses); + this.tempFileProvider = config.getTempFileProvider(); this.multiValueBind = createMultiValueBind(databasePlatform.platform()); this.deployInherit = new DeployInherit(bootupClasses); this.deployCreateProperties = new DeployCreateProperties(typeManager); @@ -517,6 +520,10 @@ SpiLogManager getLogManager() { return logManager; } + public TempFileProvider getTempFileProvider() { + return tempFileProvider; + } + private ServerCachePlugin initServerCachePlugin() { if (config.isLocalOnlyL2Cache()) { localL2Caching = true; diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java b/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java index 9177eed465..fdf77e8293 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java @@ -56,7 +56,7 @@ public final class DefaultTypeManager implements TypeManager { private final DefaultTypeFactory extraTypeFactory; - private final ScalarType fileType = new ScalarTypeFile(); + private final ScalarType fileType; private final ScalarType hstoreType = new ScalarTypePostgresHstore(); private final JsonConfig.DateTime jsonDateTime; @@ -94,6 +94,7 @@ public DefaultTypeManager(DatabaseConfig config, BootupClasses bootupClasses) { this.arrayTypeSetFactory = arrayTypeSetFactory(config.getDatabasePlatform()); this.offlineMigrationGeneration = DbOffline.isGenerateMigration(); this.defaultEnumType = config.getDefaultEnumType(); + this.fileType = new ScalarTypeFile(config.getTempFileProvider()); ServiceLoader mappers = ServiceLoader.load(ScalarJsonMapper.class); jsonMapper = mappers.findFirst().orElse(null); diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java b/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java index 5e4ad5df80..5dd6ff96e9 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; +import io.ebean.config.TempFileProvider; import io.ebean.core.type.DataBinder; import io.ebean.core.type.DataReader; import io.ebean.core.type.DocPropertyType; @@ -20,26 +21,22 @@ */ final class ScalarTypeFile extends ScalarTypeBase { - private final String prefix; - private final String suffix; - private final File directory; + private final TempFileProvider tempFileProvider; private final int bufferSize; /** - * Construct with reasonable defaults of Blob and 8096 buffer size. + * Construct with reasonable defaults of Blob and 8192 buffer size. */ - ScalarTypeFile() { - this(Types.LONGVARBINARY, "db-", null, null, 8096); + ScalarTypeFile(TempFileProvider tempFileProvider) { + this(Types.LONGVARBINARY, tempFileProvider, 8192); } /** * Create the ScalarTypeFile. */ - ScalarTypeFile(int jdbcType, String prefix, String suffix, File directory, int bufferSize) { + ScalarTypeFile(int jdbcType, TempFileProvider tempFileProvider, int bufferSize) { super(File.class, false, jdbcType); - this.prefix = prefix; - this.suffix = suffix; - this.directory = directory; + this.tempFileProvider = tempFileProvider; this.bufferSize = bufferSize; } @@ -66,7 +63,7 @@ public File read(DataReader reader) throws SQLException { } try { // stream from db into our temp file - File tempFile = File.createTempFile(prefix, suffix, directory); + File tempFile = tempFileProvider.createTempFile(); OutputStream os = getOutputStream(tempFile); pump(is, os); return tempFile; @@ -109,7 +106,7 @@ public void jsonWrite(JsonGenerator writer, File value) throws IOException { @Override public File jsonRead(JsonParser parser) throws IOException { - File tempFile = File.createTempFile(prefix, suffix, directory); + File tempFile = tempFileProvider.createTempFile(); try (OutputStream os = getOutputStream(tempFile)) { parser.readBinaryValue(os); os.flush();