diff --git a/config.js b/config.js index 046aa763a1..9a10a503b6 100644 --- a/config.js +++ b/config.js @@ -758,9 +758,19 @@ config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000; // of `manage_nsfs glacier restore` config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000; -// NSFS_GLACIER_EXPIRY_INTERVAL indicates the interval between runs -// of `manage_nsfs glacier expiry` -config.NSFS_GLACIER_EXPIRY_INTERVAL = 12 * 60 * 60 * 1000; +// NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies +// when NooBaa should allow running glacier expiry process +// NOTE: This will also be in the same timezone as specified in +// NSFS_GLACIER_EXPIRY_TZ +config.NSFS_GLACIER_EXPIRY_RUN_TIME = '03:00'; + +// NSFS_GLACIER_EXPIRY_RUN_TIME_TOLERANCE_MINS configures the delay +// tolerance in minutes. +// +// eg. If the expiry run time is set to 03:00 and the tolerance is +// set to be 2 mins then the expiry can trigger till 03:02 (unless +// already triggered between 03:00 - 03:02 +config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS = 2 * 60; /** @type {'UTC' | 'LOCAL'} */ config.NSFS_GLACIER_EXPIRY_TZ = 'LOCAL'; diff --git a/src/manage_nsfs/manage_nsfs_glacier.js b/src/manage_nsfs/manage_nsfs_glacier.js index 9352259d15..eef249532c 100644 --- a/src/manage_nsfs/manage_nsfs_glacier.js +++ b/src/manage_nsfs/manage_nsfs_glacier.js @@ -69,12 +69,65 @@ async function process_expiry() { const fs_context = native_fs_utils.get_process_fs_context(); await lock_and_run(fs_context, SCAN_LOCK, async () => { - if (!(await time_exceeded(fs_context, config.NSFS_GLACIER_EXPIRY_INTERVAL, GlacierBackend.EXPIRY_TIMESTAMP_FILE))) return; + const backend = getGlacierBackend(); + if ( + await backend.low_free_space() || + await is_desired_time( + fs_context, + new Date(), + config.NSFS_GLACIER_EXPIRY_RUN_TIME, + config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS, + GlacierBackend.EXPIRY_TIMESTAMP_FILE, + ) + ) { + await backend.expiry(fs_context); + await record_current_time(fs_context, GlacierBackend.EXPIRY_TIMESTAMP_FILE); + } + }); +} +/** + * is_desired_time returns true if the given time matches with + * the desired time or if + * @param {nb.NativeFSContext} fs_context + * @param {Date} current + * @param {string} desire time in format 'hh:mm' + * @param {number} delay_limit_mins + * @param {string} timestamp_file + * @returns {Promise} + */ +async function is_desired_time(fs_context, current, desire, delay_limit_mins, timestamp_file) { + const [desired_hour, desired_min] = desire.split(':').map(Number); + if ( + isNaN(desired_hour) || + isNaN(desired_min) || + (desired_hour < 0 || desired_hour >= 24) || + (desired_min < 0 || desired_min >= 60) + ) { + throw new Error('invalid desired_time - must be hh:mm'); + } - await getGlacierBackend().expiry(fs_context); - await record_current_time(fs_context, GlacierBackend.EXPIRY_TIMESTAMP_FILE); - }); + const min_time = get_tz_date(desired_hour, desired_min, 0, config.NSFS_GLACIER_EXPIRY_TZ); + const max_time = get_tz_date(desired_hour, desired_min + delay_limit_mins, 0, config.NSFS_GLACIER_EXPIRY_TZ); + + if (current >= min_time && current <= max_time) { + try { + const { data } = await nb_native().fs.readFile(fs_context, path.join(config.NSFS_GLACIER_LOGS_DIR, timestamp_file)); + const lastrun = new Date(data.toString()); + + // Last run should NOT be in this window + if (lastrun >= min_time && lastrun <= max_time) return false; + } catch (error) { + if (error.code === 'ENOENT') return true; + console.error('failed to read last run timestamp:', error, 'timestamp_file:', timestamp_file); + + throw error; + } + + return true; + } + + return false; } /** @@ -134,6 +187,31 @@ async function run_glacier_operation(fs_context, log_namespace, cb) { } } +/** + * @param {number} hours + * @param {number} mins + * @param {number} secs + * @param {'UTC' | 'LOCAL'} tz + * @returns {Date} + */ +function get_tz_date(hours, mins, secs, tz) { + const date = new Date(); + + if (tz === 'UTC') { + date.setUTCHours(hours); + date.setUTCMinutes(hours); + date.setUTCSeconds(secs); + date.setUTCMilliseconds(0); + } else { + date.setHours(hours); + date.setMinutes(mins); + date.setSeconds(secs); + date.setMilliseconds(0); + } + + return date; +} + /** * lock_and_run acquires a flock and calls the given callback after * acquiring the lock diff --git a/src/sdk/nsfs_glacier_backend/tapecloud.js b/src/sdk/nsfs_glacier_backend/tapecloud.js index 8ff3553fe7..e9d2956437 100644 --- a/src/sdk/nsfs_glacier_backend/tapecloud.js +++ b/src/sdk/nsfs_glacier_backend/tapecloud.js @@ -278,7 +278,7 @@ class TapeCloudGlacierBackend extends GlacierBackend { async low_free_space() { const result = await exec(get_bin_path(LOW_FREE_SPACE_SCRIPT), { return_stdout: true }); - return result.toLowerCase() === 'true'; + return result.toLowerCase().trim() === 'true'; } // ============= PRIVATE FUNCTIONS =============