diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java index 871ab5bc9a..415995d815 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java @@ -102,6 +102,12 @@ public class AmoroManagementConf { .defaultValue(10) .withDescription("The number of threads used for orphan files cleaning."); + public static final ConfigOption CLEAN_ORPHAN_FILES_INTERVAL = + ConfigOptions.key("clean-orphan-files.interval") + .durationType() + .defaultValue(Duration.ofDays(1)) + .withDescription("Interval for cleaning orphan files."); + public static final ConfigOption CLEAN_DANGLING_DELETE_FILES_ENABLED = ConfigOptions.key("clean-dangling-delete-files.enabled") .booleanType() diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java index 09f7a4d3c2..a45385026e 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java @@ -49,7 +49,9 @@ public void setup(TableManager tableManager, Configurations conf) { if (conf.getBoolean(AmoroManagementConf.CLEAN_ORPHAN_FILES_ENABLED)) { this.orphanFilesCleaningExecutor = new OrphanFilesCleaningExecutor( - tableManager, conf.getInteger(AmoroManagementConf.CLEAN_ORPHAN_FILES_THREAD_COUNT)); + tableManager, + conf.getInteger(AmoroManagementConf.CLEAN_ORPHAN_FILES_THREAD_COUNT), + conf.get(AmoroManagementConf.CLEAN_ORPHAN_FILES_INTERVAL)); } if (conf.getBoolean(AmoroManagementConf.CLEAN_DANGLING_DELETE_FILES_ENABLED)) { this.danglingDeleteFilesCleaningExecutor = diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java index 9d40090ad1..6d6cf01dc2 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java @@ -26,18 +26,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; + public class OrphanFilesCleaningExecutor extends BaseTableExecutor { private static final Logger LOG = LoggerFactory.getLogger(OrphanFilesCleaningExecutor.class); + private final Duration interval; - private static final long INTERVAL = 24 * 60 * 60 * 1000L; - - public OrphanFilesCleaningExecutor(TableManager tableRuntimes, int poolSize) { + public OrphanFilesCleaningExecutor(TableManager tableRuntimes, int poolSize, Duration interval) { super(tableRuntimes, poolSize); + this.interval = interval; } @Override protected long getNextExecutingTime(TableRuntime tableRuntime) { - return INTERVAL; + return interval.toMillis(); } @Override diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java index debe3f0cae..9f2aeca5d4 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java @@ -173,6 +173,7 @@ private TableProperties() {} public static final String MIN_ORPHAN_FILE_EXISTING_TIME = "clean-orphan-file.min-existing-time-minutes"; public static final long MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT = 2880; // 2 Days + public static final String ENABLE_DANGLING_DELETE_FILES_CLEAN = "clean-dangling-delete-files.enabled"; public static final boolean ENABLE_DANGLING_DELETE_FILES_CLEAN_DEFAULT = true; diff --git a/dist/src/main/amoro-bin/conf/config.yaml b/dist/src/main/amoro-bin/conf/config.yaml index d16027c349..ae1de24e48 100644 --- a/dist/src/main/amoro-bin/conf/config.yaml +++ b/dist/src/main/amoro-bin/conf/config.yaml @@ -66,6 +66,7 @@ ams: clean-orphan-files: enabled: true thread-count: 10 + interval: 1d clean-dangling-delete-files: enabled: true