filter out failed recalls

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> update comments Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> fix linting issue Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> revamp restore code and add tests Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> add more tests Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> add cleanup to the test Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> address self review comments Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> fix partial failure of finalize_restore Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> improving task_show output parsing Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> fix parsing edge case Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>
noobaa · May 23, 2024 · 132d4e6 · 132d4e6
1 parent 8d1652c
commit 132d4e6
Show file tree

Hide file tree

Showing 4 changed files with 439 additions and 177 deletions.
diff --git a/config.js b/config.js
@@ -762,6 +762,11 @@ config.NSFS_GLACIER_BACKEND = 'TAPECLOUD';
 // TAPECLOUD Glacier backend specific configs
 config.NSFS_GLACIER_TAPECLOUD_BIN_DIR = '/opt/ibm/tapecloud/bin';
 
+// If set to true will disable cleanup of the task show output
+// Should be used only for debugging or else will keep filling
+// up the disk space.
+config.NSFS_GLACIER_TAPECLOUD_PRESERVE_TASK_SHOW_OUTPUT = false;
+
 // NSFS_GLACIER_MIGRATE_INTERVAL indicates the interval between runs
 // of `manage_nsfs glacier migrate`
 config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000;

diff --git a/src/sdk/nsfs_glacier_backend/backend.js b/src/sdk/nsfs_glacier_backend/backend.js
@@ -17,9 +17,9 @@ class GlacierBackend {
     static EXPIRY_TIMESTAMP_FILE = 'expiry.timestamp';
 
     /**
-     * XATTR_RESTORE_REQUEST is set to a NUMBER (expiry days) by `restore_object` when 
-     * a restore request is made. This is unset by the underlying restore process when 
-     * it finishes the request, this  is to ensure that the same object is not queued 
+     * XATTR_RESTORE_REQUEST is set to a NUMBER (expiry days) by `restore_object` when
+     * a restore request is made. This is unset by the underlying restore process when
+     * it finishes the request, this  is to ensure that the same object is not queued
      * for restoration multiple times.
      */
     static XATTR_RESTORE_REQUEST = 'user.noobaa.restore.request';
@@ -29,7 +29,7 @@ class GlacierBackend {
      * NooBaa (in case restore is issued again while the object is on disk).
      * This is read by the underlying "disk evict" process to determine if the object
      * should be evicted from the disk or not.
-     * 
+     *
      * NooBaa will use this date to determine if the object is on disk or not, if the
      * expiry date is in the future, the object is on disk, if the expiry date is in
      * the past, the object is not on disk. This may or may not represent the actual
@@ -54,10 +54,10 @@ class GlacierBackend {
      * migrate must take a file name which will have newline seperated
      * entries of filenames which needs to be migrated to GLACIER and
      * should perform migration of those files if feasible.
-     * 
+     *
      * The function should return false if it needs the log file to be
      * preserved.
-     * 
+     *
      * NOTE: This needs to be implemented by each backend.
      * @param {nb.NativeFSContext} fs_context
      * @param {string} log_file log filename
@@ -72,10 +72,10 @@ class GlacierBackend {
      * restore must take a file name which will have newline seperated
      * entries of filenames which needs to be restored from GLACIER and
      * should perform restore of those files if feasible
-     * 
+     *
      * The function should return false if it needs the log file to be
      * preserved.
-     * 
+     *
      * NOTE: This needs to be implemented by each backend.
      * @param {nb.NativeFSContext} fs_context
      * @param {string} log_file log filename
@@ -88,7 +88,7 @@ class GlacierBackend {
 
     /**
      * expiry moves the restored files back to glacier
-     * 
+     *
      * NOTE: This needs to be implemented by each backend.
      * @param {nb.NativeFSContext} fs_context
      */
@@ -99,13 +99,13 @@ class GlacierBackend {
     /**
      * low_free_space must return true if the backend has
      * low free space.
-     * 
+     *
      * NOTE: This may be used as a precheck before executing
      * operations like `migrate` and `restore`.
-     * 
+     *
      * Example: `migrate` can be more frequently if this function
      * returns `true`.
-     * 
+     *
      * @returns {Promise<boolean>}
      */
     async low_free_space() {
@@ -114,7 +114,7 @@ class GlacierBackend {
 
     /**
      * should_migrate returns true if the given file must be migrated
-     * 
+     *
      * The caller can pass the stat data, if none is passed, stat is
      * called internally.
      * @param {string} file name of the file
@@ -145,12 +145,12 @@ class GlacierBackend {
     /**
      * get_restore_status returns status of the object at the given
      * file_path
-     * 
+     *
      * NOTE: Returns undefined if `user.storage_class` attribute is not
      * `GLACIER`
-     * @param {nb.NativeFSXattr} xattr 
-     * @param {Date} now 
-     * @param {string} file_path 
+     * @param {nb.NativeFSXattr} xattr
+     * @param {Date} now
+     * @param {string} file_path
      * @returns {nb.RestoreStatus | undefined}
      */
     static get_restore_status(xattr, now, file_path) {
@@ -207,7 +207,7 @@ class GlacierBackend {
      * @param {Date} from
      * @param {Number} days - float
      * @param {string} desired_date_time - in format HH:MM:SS
-     * @param {'UTC' | 'LOCAL'} tz 
+     * @param {'UTC' | 'LOCAL'} tz
      * @returns {Date}
      */
     static generate_expiry(from, days, desired_date_time, tz) {
@@ -243,7 +243,7 @@ class GlacierBackend {
 
     /**
      * should_restore returns true if the give file must be restored
-     * 
+     *
      * The caller can pass the stat data, if none is passed, stat is
      * called internally.
      * @param {string} file name of the file
@@ -263,6 +263,14 @@ class GlacierBackend {
         const restore_status = GlacierBackend.get_restore_status(stat.xattr, new Date(), file);
         if (!restore_status) return false;
 
+        // We don't check for pre-existing expiry here, it can happen in 2 cases
+        // 1. A restore is already going and someone somehow initiated this second
+        // call. In that case we might see partial extended attributes such that
+        // both request as well a future expiry time exists.
+        // 2. A restore request was partially processed and then failed before
+        // removing the request extended attribute. In such case, NSFS would still
+        // report the object restore status to be `ONGOING` and we are going
+        // to allow a retry of that entry.
         return restore_status.state === GlacierBackend.RESTORE_STATUS_ONGOING;
     }
 }