diff --git a/src/cmd/health.js b/src/cmd/health.js index 9bb0e52fe9..b10d8c2da2 100644 --- a/src/cmd/health.js +++ b/src/cmd/health.js @@ -92,7 +92,7 @@ const fork_response_code = { const health_errors_tyes = { PERSISTENT: 'PERSISTENT', TEMPORARY: 'TEMPORARY', -} +}; //suppress aws sdk related commands. process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = '1'; @@ -180,7 +180,7 @@ class NSFSHealth { } } }); - } catch(err) { + } catch (err) { console.log('Error while pinging endpoint host :' + HOSTNAME + ', port ' + this.https_port, err); return { response: fork_response_code.NOT_RUNNING.response_code, diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index fd363c9446..bbf178c2e7 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -12,6 +12,7 @@ const nb_native = require('../util/nb_native'); const cloud_utils = require('../util/cloud_utils'); const string_utils = require('../util/string_utils'); const native_fs_utils = require('../util/native_fs_utils'); +const mongo_utils = require('../util/mongo_utils'); const SensitiveString = require('../util/sensitive_string'); const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; @@ -52,7 +53,7 @@ async function check_and_create_config_dirs() { for (const dir_path of pre_req_dirs) { try { const fs_context = native_fs_utils.get_process_fs_context(); - const dir_exists = await native_fs_utils.config_file_exists(fs_context, dir_path); + const dir_exists = await native_fs_utils.is_path_exists(fs_context, dir_path); if (dir_exists) { dbg.log1('nsfs.check_and_create_config_dirs: config dir exists:', dir_path); } else { @@ -142,7 +143,7 @@ async function fetch_bucket_data(argv, from_file) { data.s3_policy = JSON.parse(argv.bucket_policy.toString()); } } - if (action === ACTIONS.UPDATE) { + if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) { data = _.omitBy(data, _.isUndefined); data = await fetch_existing_bucket_data(data); } @@ -188,9 +189,9 @@ async function add_bucket(data) { await verify_bucket_owner(data.bucket_owner.unwrap()); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); const bucket_conf_path = get_config_file_path(buckets_dir_path, data.name); - const exists = await native_fs_utils.config_file_exists(fs_context, bucket_conf_path); + const exists = await native_fs_utils.is_path_exists(fs_context, bucket_conf_path); if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name.unwrap()); - + data._id = mongo_utils.mongoObjectId(); const data_json = JSON.stringify(data); // We take an object that was stringify // (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters) @@ -266,7 +267,7 @@ async function update_bucket(data) { const cur_bucket_config_path = get_config_file_path(buckets_dir_path, cur_name.unwrap()); const new_bucket_config_path = get_config_file_path(buckets_dir_path, data.name.unwrap()); - const exists = await native_fs_utils.config_file_exists(fs_context, new_bucket_config_path); + const exists = await native_fs_utils.is_path_exists(fs_context, new_bucket_config_path); if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name.unwrap()); data = JSON.stringify(_.omit(data, ['new_name'])); @@ -281,13 +282,15 @@ async function update_bucket(data) { async function delete_bucket(data) { await validate_bucket_args(data, ACTIONS.DELETE); - const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); const bucket_config_path = get_config_file_path(buckets_dir_path, data.name); try { + const bucket_temp_dir_path = path.join(data.path, config.NSFS_TEMP_DIR_NAME + "_" + data._id); + await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context, true); await native_fs_utils.delete_config_file(fs_context, buckets_dir_path, bucket_config_path); } catch (err) { if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name); + throw err; } write_stdout_response(ManageCLIResponse.BucketDeleted); } @@ -440,14 +443,14 @@ async function add_account(data) { const account_config_path = get_config_file_path(accounts_dir_path, data.name); const account_config_access_key_path = get_symlink_config_file_path(access_keys_dir_path, access_key); - const name_exists = await native_fs_utils.config_file_exists(fs_context, account_config_path); - const access_key_exists = await native_fs_utils.config_file_exists(fs_context, account_config_access_key_path, true); + const name_exists = await native_fs_utils.is_path_exists(fs_context, account_config_path); + const access_key_exists = await native_fs_utils.is_path_exists(fs_context, account_config_access_key_path, true); if (name_exists || access_key_exists) { const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists; throw_cli_error(err_code); } - + data._id = mongo_utils.mongoObjectId(); data = JSON.stringify(data); // We take an object that was stringify // (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters) @@ -487,8 +490,8 @@ async function update_account(data) { const new_account_config_path = get_config_file_path(accounts_dir_path, data.name.unwrap()); const cur_access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, cur_access_key.unwrap()); const new_access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key.unwrap()); - const name_exists = update_name && await native_fs_utils.config_file_exists(fs_context, new_account_config_path); - const access_key_exists = update_access_key && await native_fs_utils.config_file_exists(fs_context, new_access_key_config_path, true); + const name_exists = update_name && await native_fs_utils.is_path_exists(fs_context, new_account_config_path); + const access_key_exists = update_access_key && await native_fs_utils.is_path_exists(fs_context, new_access_key_config_path, true); if (name_exists || access_key_exists) { const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists; throw_cli_error(err_code); @@ -673,7 +676,7 @@ async function validate_bucket_args(data, action) { if (is_undefined(data.system_owner)) throw_cli_error(ManageCLIError.MissingBucketEmailFlag); if (!data.path) throw_cli_error(ManageCLIError.MissingBucketPathFlag); const fs_context = native_fs_utils.get_process_fs_context(); - const exists = await native_fs_utils.config_file_exists(fs_context, data.path); + const exists = await native_fs_utils.is_path_exists(fs_context, data.path); if (!exists) { throw_cli_error(ManageCLIError.InvalidStoragePath, data.path); } @@ -741,7 +744,7 @@ async function validate_account_args(data, action) { return; } const fs_context = native_fs_utils.get_process_fs_context(); - const exists = await native_fs_utils.config_file_exists(fs_context, data.nsfs_account_config.new_buckets_path); + const exists = await native_fs_utils.is_path_exists(fs_context, data.nsfs_account_config.new_buckets_path); if (!exists) { throw_cli_error(ManageCLIError.InvalidAccountNewBucketsPath, data.nsfs_account_config.new_buckets_path); } diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js index aaf5574c6f..75c5939480 100644 --- a/src/sdk/bucketspace_fs.js +++ b/src/sdk/bucketspace_fs.js @@ -14,6 +14,7 @@ const _ = require('lodash'); const util = require('util'); const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils'); const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils'); +const mongo_utils = require('../util/mongo_utils'); const KeysSemaphore = require('../util/keys_semaphore'); const native_fs_utils = require('../util/native_fs_utils'); @@ -344,6 +345,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { new_bucket_defaults(account, { name, tag, lock_enabled, force_md5_etag }, create_uls, bucket_storage_path) { return { + _id: mongo_utils.mongoObjectId(), name, tag: js_utils.default_value(tag, undefined), owner_account: account._id, @@ -378,10 +380,15 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { full_path: path.join(this.fs_root, namespace_bucket_config.write_resource.path) // includes write_resource.path + bucket name (s3 flow) }, object_sdk); } else if (namespace_bucket_config) { + // S3 Delete for NSFS Manage buckets const list = await ns.list_objects({ ...params, limit: 1 }, object_sdk); if (list && list.objects && list.objects.length > 0) { throw new RpcError('NOT_EMPTY', 'underlying directory has files in it'); } + const bucket = await object_sdk.read_bucket_sdk_config_info(params.name) + const bucket_temp_dir_path = path.join(namespace_bucket_config.write_resource.path, + config.NSFS_TEMP_DIR_NAME + "_" + bucket._id); + await native_fs_utils.folder_delete(bucket_temp_dir_path, this.fs_context, true); } dbg.log1(`BucketSpaceFS: delete_fs_bucket ${bucket_path}`); // delete bucket config json file diff --git a/src/sdk/namespace_fs.js b/src/sdk/namespace_fs.js index a318163630..811c5e1205 100644 --- a/src/sdk/namespace_fs.js +++ b/src/sdk/namespace_fs.js @@ -1702,7 +1702,7 @@ class NamespaceFS { await target_file.close(fs_context); target_file = null; - if (config.NSFS_REMOVE_PARTS_ON_COMPLETE) await this._folder_delete(params.mpu_path, fs_context); + if (config.NSFS_REMOVE_PARTS_ON_COMPLETE) await native_fs_utils.folder_delete(params.mpu_path, fs_context); return upload_info; } catch (err) { dbg.error(err); @@ -1733,7 +1733,7 @@ class NamespaceFS { const fs_context = this.prepare_fs_context(object_sdk); await this._load_multipart(params, fs_context); dbg.log0('NamespaceFS: abort_object_upload', params.mpu_path); - await this._folder_delete(params.mpu_path, fs_context); + await native_fs_utils.folder_delete(params.mpu_path, fs_context); } /////////////////// @@ -2362,21 +2362,6 @@ class NamespaceFS { } } - async _folder_delete(dir, fs_context) { - const entries = await nb_native().fs.readdir(fs_context, dir); - const results = await Promise.all(entries.map(entry => { - const fullPath = path.join(dir, entry.name); - const task = native_fs_utils.isDirectory(entry) ? this._folder_delete(fullPath, fs_context) : - nb_native().fs.unlink(fs_context, fullPath); - return task.catch(error => ({ error })); - })); - results.forEach(result => { - // Ignore missing files/directories; bail on other errors - if (result && result.error && result.error.code !== 'ENOENT') throw result.error; - }); - await nb_native().fs.rmdir(fs_context, dir); - } - async create_uls(params, object_sdk) { const fs_context = this.prepare_fs_context(object_sdk); dbg.log0('NamespaceFS: create_uls fs_context:', fs_context, 'new_dir_path: ', params.full_path); @@ -2398,7 +2383,7 @@ class NamespaceFS { throw new RpcError('NOT_EMPTY', 'underlying directory has files in it'); } - await this._folder_delete(params.full_path, fs_context); + await native_fs_utils.folder_delete(params.full_path, fs_context); } catch (err) { throw this._translate_object_error_codes(err); } diff --git a/src/server/system_services/schemas/nsfs_account_schema.js b/src/server/system_services/schemas/nsfs_account_schema.js index feb0c08f94..12c43e9fa0 100644 --- a/src/server/system_services/schemas/nsfs_account_schema.js +++ b/src/server/system_services/schemas/nsfs_account_schema.js @@ -5,6 +5,7 @@ module.exports = { $id: 'account_schema', type: 'object', required: [ + '_id', 'name', 'email', 'access_keys', @@ -13,6 +14,9 @@ module.exports = { 'allow_bucket_creation', ], properties: { + _id: { + type: 'string', + }, name: { type: 'string' }, diff --git a/src/server/system_services/schemas/nsfs_bucket_schema.js b/src/server/system_services/schemas/nsfs_bucket_schema.js index 9cecb6f3b3..2c4f5b242c 100644 --- a/src/server/system_services/schemas/nsfs_bucket_schema.js +++ b/src/server/system_services/schemas/nsfs_bucket_schema.js @@ -5,6 +5,7 @@ module.exports = { $id: 'bucket_schema', type: 'object', required: [ + '_id', 'name', 'system_owner', 'bucket_owner', @@ -14,6 +15,12 @@ module.exports = { 'creation_date', ], properties: { + _id: { + type: 'string', + }, + owner_account: { + type: 'string', + }, name: { type: 'string', }, diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js index f768ae1b52..958a54b5c2 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js @@ -45,6 +45,7 @@ describe('manage nsfs cli account flow', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); const defaults = { + _id: 'account1', type: 'account', name: 'account1', email: 'account1@noobaa.io', @@ -151,6 +152,7 @@ describe('manage nsfs cli account flow', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs1'); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs1/'); const defaults = { + _id: 'account1', type: 'account', name: 'account1', email: 'account1@noobaa.io', @@ -255,6 +257,7 @@ describe('manage nsfs cli account flow', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs1'); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs1/'); const defaults = [{ + _id: 'account1', type: 'account', name: 'account1', email: 'account1@noobaa.io', @@ -264,6 +267,7 @@ describe('manage nsfs cli account flow', () => { access_key: 'GIGiFAnjaaE7OKD5N7hA', secret_key: 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE', }, { + _id: 'account1', type: 'account', name: 'account2', email: 'account2@noobaa.io', @@ -273,6 +277,7 @@ describe('manage nsfs cli account flow', () => { access_key: 'BIBiFAnjaaE7OKD5N7hA', secret_key: 'BIBYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE', }, { + _id: 'account1', type: 'account', name: 'account3', email: 'account3@noobaa.io', diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js index 096ec06868..a061c2317a 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js @@ -273,6 +273,23 @@ describe('schema validation NC NSFS account', () => { assert_validation(account_data, reason, message); }); + it('account without _id', () => { + const account_data = get_account_data(); + delete account_data._id; + const reason = 'Test should have failed because of missing required property ' + + '_id'; + const message = "must have required property '_id'"; + assert_validation(account_data, reason, message); + }); + + it('account with undefined _id', () => { + const account_data = get_account_data(); + account_data._id = undefined; + const reason = 'Test should have failed because of missing required property ' + + '_id'; + const message = "must have required property '_id'"; + assert_validation(account_data, reason, message); + }); }); describe('account with wrong types', () => { @@ -312,7 +329,6 @@ describe('schema validation NC NSFS account', () => { const message = 'must be equal to one of the allowed values'; assert_validation(account_data, reason, message); }); - }); }); @@ -320,6 +336,7 @@ describe('schema validation NC NSFS account', () => { function get_account_data() { const account_name = 'account1'; + const id = '65a62e22ceae5e5f1a758aa9'; const account_email = 'account1@noobaa.io'; const access_key = 'GIGiFAnjaaE7OKD5N7hA'; const secret_key = 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE'; @@ -330,6 +347,7 @@ function get_account_data() { }; const account_data = { + _id: id, name: account_name, email: account_email, access_keys: [{ diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js new file mode 100644 index 0000000000..df9627bbf0 --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js @@ -0,0 +1,146 @@ +/* Copyright (C) 2016 NooBaa */ +/* eslint-disable no-undef */ +'use strict'; + +// disabling init_rand_seed as it takes longer than the actual test execution +process.env.DISABLE_INIT_RANDOM_SEED = "true"; + +const _ = require('lodash'); +const path = require('path'); +const P = require('../../../util/promise'); +const fs_utils = require('../../../util/fs_utils'); +const os_util = require('../../../util/os_utils'); +const config_module = require('../../../../config'); +const native_fs_utils = require('../../../util/native_fs_utils'); + +const MAC_PLATFORM = 'darwin'; +let tmp_fs_path = '/tmp/test_bucketspace_fs'; +if (process.platform === MAC_PLATFORM) { + tmp_fs_path = '/private/' + tmp_fs_path; +} +let bucket_storage_path; +let bucket_temp_dir_path; + +const DEFAULT_FS_CONFIG = { + uid: process.getuid(), + gid: process.getgid(), + backend: '', + warn_threshold_ms: 100, +}; + +const nc_nsfs_manage_actions = { + ADD: 'add', + UPDATE: 'update', + LIST: 'list', + DELETE: 'delete', + STATUS: 'status', +}; + +// eslint-disable-next-line max-lines-per-function +describe('manage nsfs cli account flow', () => { + const buckets_schema_dir = 'buckets'; + + describe('cli delete bucket', () => { + const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs2'); + const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs2/'); + bucket_storage_path = path.join(tmp_fs_path, 'root_path_manage_nsfs2', 'bucket1'); + + const account_defaults = { + _id: '65a9fbe7ab49fd2fe430bc3f', + type: 'account', + name: 'account_test', + email: 'account1@noobaa.io', + new_buckets_path: `${root_path}new_buckets_path_user1/`, + uid: 999, + gid: 999, + access_key: 'GIGiFAnjaaE7OKD5N7hX', + secret_key: 'G2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE', + creation_date: '2024-01-19T04:34:47.273Z', + }; + + const bucket_defaults = { + type: 'bucket', + name: 'bucket1', + tag:'', + owner_account: 'account_test', + system_owner: 'account1@noobaa.io', + bucket_owner: 'account1@noobaa.io', + email: 'account1@noobaa.io', + path: bucket_storage_path, + should_create_underlying_storage: true, + creation_date: '2024-01-19T04:34:47.273Z', + }; + + beforeEach(async () => { + await P.all(_.map([buckets_schema_dir], async dir => + fs_utils.create_fresh_path(`${config_root}/${dir}`))); + await fs_utils.create_fresh_path(root_path); + await fs_utils.create_fresh_path(bucket_storage_path); + const action = nc_nsfs_manage_actions.ADD; + // Account add + const { type: account_type, new_buckets_path: account_path } = account_defaults; + const account_options = { config_root, ...account_defaults }; + await fs_utils.create_fresh_path(account_path); + await fs_utils.file_must_exist(account_path); + await exec_manage_cli(account_type, action, account_options); + + //bucket add + const { type: bucket_type, path: bucket_path } = bucket_defaults; + const bucket_options = { config_root, ...bucket_defaults }; + await fs_utils.create_fresh_path(bucket_path); + await fs_utils.file_must_exist(bucket_path); + const resp = await exec_manage_cli(bucket_type, action, bucket_options); + const bucket_resp = JSON.parse(resp); + expect(bucket_resp.response.reply._id).not.toBeNull(); + //create temp dir + bucket_temp_dir_path = path.join(bucket_storage_path, + config_module.NSFS_TEMP_DIR_NAME + "_" + bucket_resp.response.reply._id); + await fs_utils.create_fresh_path(bucket_temp_dir_path); + await fs_utils.file_must_exist(bucket_temp_dir_path); + }); + + afterEach(async () => { + await fs_utils.folder_delete(`${config_root}`); + await fs_utils.folder_delete(`${root_path}`); + }); + + it('cli delete bucket and delete temp dir', async () => { + let is_path_exists = await native_fs_utils.is_path_exists(DEFAULT_FS_CONFIG, bucket_temp_dir_path); + expect(is_path_exists).toBe(true); + const account_options = { config_root, name: 'bucket1'}; + const action = nc_nsfs_manage_actions.DELETE; + await exec_manage_cli('bucket', action, account_options); + is_path_exists = await native_fs_utils.is_path_exists(DEFAULT_FS_CONFIG, bucket_temp_dir_path); + expect(is_path_exists).toBe(false); + }); + }); +}) + +/** + * exec_manage_cli will get the flags for the cli and runs the cli with it's flags + * @param {string} type + * @param {string} action + * @param {object} options + */ +async function exec_manage_cli(type, action, options) { + let account_flags = ``; + for (const key in options) { + if (Object.hasOwn(options, key)) { + if (typeof options[key] === 'boolean') { + account_flags += `--${key} `; + } else { + account_flags += `--${key} ${options[key]} `; + } + } + } + account_flags = account_flags.trim(); + + const command = `node src/cmd/manage_nsfs ${type} ${action} ${account_flags}`; + let res; + try { + res = await os_util.exec(command, { return_stdout: true }); + } catch (e) { + res = e; + } + return res; +} \ No newline at end of file diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js index 6dd294962b..42ad921a31 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js @@ -263,6 +263,24 @@ describe('schema validation NC NSFS bucket', () => { assert_validation(bucket_data, reason, message); }); + it('bucket with undefined _id', () => { + const bucket_data = get_bucket_data(); + bucket_data._id = undefined; + const reason = 'Test should have failed because of missing required property ' + + '_id'; + const message = "must have required property '_id'"; + assert_validation(bucket_data, reason, message); + }); + + it('bucket without _id', () => { + const bucket_data = get_bucket_data(); + delete bucket_data._id; + const reason = 'Test should have failed because of missing required property ' + + '_id'; + const message = "must have required property '_id'"; + assert_validation(bucket_data, reason, message); + }); + }); describe('bucket with wrong types', () => { @@ -343,6 +361,7 @@ describe('schema validation NC NSFS bucket', () => { function get_bucket_data() { const bucket_name = 'bucket1'; + const id = '65a62e22ceae5e5f1a758aa8'; const system_owner = 'account1@noobaa.io'; const bucket_owner = 'account1@noobaa.io'; const versioning_disabled = 'DISABLED'; @@ -350,6 +369,7 @@ function get_bucket_data() { const path = '/tmp/nsfs_root1'; const bucket_data = { + _id: id, name: bucket_name, system_owner: system_owner, bucket_owner: bucket_owner, diff --git a/src/test/unit_tests/test_bucketspace_fs.js b/src/test/unit_tests/test_bucketspace_fs.js index bbedb9bba3..ec949f35ab 100644 --- a/src/test/unit_tests/test_bucketspace_fs.js +++ b/src/test/unit_tests/test_bucketspace_fs.js @@ -19,9 +19,11 @@ const nb_native = require('../../util/nb_native'); const MAC_PLATFORM = 'darwin'; const test_bucket = 'bucket1'; -const test_not_empty_bucketbucket = 'notemptybucket'; +const test_not_empty_bucket = 'notemptybucket'; +const test_bucket_temp_dir = 'buckettempdir'; const test_bucket_invalid = 'bucket_invalid'; let tmp_fs_path = '/tmp/test_bucketspace_fs'; +let bucket_temp_id; if (process.platform === MAC_PLATFORM) { tmp_fs_path = '/private/' + tmp_fs_path; } @@ -51,6 +53,7 @@ const DEFAULT_FS_CONFIG = { // since the account in NS NSFS should be valid to the nsfs_account_schema // had to remove additional properties: has_s3_access: 'true' and nsfs_only: 'true' const account_user1 = { + _id: '65a8edc9bc5d5bbf9db71b91', name: 'user1', email: 'user1@noobaa.io', allow_bucket_creation: true, @@ -67,6 +70,7 @@ const account_user1 = { }; const account_user2 = { + _id: '65a8edc9bc5d5bbf9db71b92', name: 'user2', email: 'user2@noobaa.io', allow_bucket_creation: true, @@ -82,6 +86,7 @@ const account_user2 = { }; const account_user3 = { + _id: '65a8edc9bc5d5bbf9db71b93', name: 'user3', email: 'user3@noobaa.io', allow_bucket_creation: true, @@ -132,6 +137,11 @@ function make_dummy_object_sdk() { read_bucket_sdk_namespace_info(name) { dummy_ns.write_resource.path = path.join(new_buckets_path, name.toString()); dummy_ns.read_resources[0].resource.name = name.toString(); + if (name === test_bucket_temp_dir) { + dummy_ns.should_create_underlying_storage = false; + } else { + dummy_ns.should_create_underlying_storage = true; + } return dummy_ns; }, _get_bucket_namespace(name) { @@ -142,6 +152,12 @@ function make_dummy_object_sdk() { is_nsfs_bucket(ns) { const fs_root_path = ns?.write_resource?.resource?.fs_root_path; return Boolean(fs_root_path || fs_root_path === ''); + }, + read_bucket_sdk_config_info(name) { + return { + _id : bucket_temp_id, + name: name + } } }; } @@ -237,9 +253,9 @@ mocha.describe('bucketspace_fs', function() { }); mocha.it('validate bucket access with default context', async function() { try { - const param = { name: test_bucket}; - const invalid_objects = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, path.join(new_buckets_path, param.name)); - assert.equal(invalid_objects.length, 0); + const param = { name: test_bucket}; + const invalid_objects = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, path.join(new_buckets_path, param.name)); + assert.equal(invalid_objects.length, 0); } catch (err) { assert.ok(err.code === 'EACCES'); assert.ok(err.message === 'Permission denied'); @@ -313,7 +329,7 @@ mocha.describe('bucketspace_fs', function() { } }); mocha.it('delete_bucket for non empty buckets', async function() { - const param = { name: test_not_empty_bucketbucket}; + const param = { name: test_not_empty_bucket}; await create_bucket(param.name); const bucket_file_path = path.join(new_buckets_path, param.name, 'dummy.txt'); await nb_native().fs.writeFile(ACCOUNT_FS_CONFIG, bucket_file_path, @@ -328,6 +344,33 @@ mocha.describe('bucketspace_fs', function() { assert.equal(err.message, 'underlying directory has files in it'); } }); + + mocha.it('delete_bucket for should_create_underlying_storage false', async function() { + const param = { name: test_bucket_temp_dir}; + await create_bucket(param.name); + await fs.promises.stat(path.join(new_buckets_path, param.name)); + const bucket_config_path = get_config_file_path(buckets, param.name); + const data = await fs.promises.readFile(bucket_config_path); + const bucket = await JSON.parse(data.toString()); + bucket_temp_id = bucket._id; + const bucket_temp_dir_path = path.join(new_buckets_path, param.name, config.NSFS_TEMP_DIR_NAME + "_" + bucket._id); + await nb_native().fs.mkdir(ACCOUNT_FS_CONFIG, bucket_temp_dir_path); + await fs.promises.stat(bucket_temp_dir_path); + await bucketspace_fs.delete_bucket(param, dummy_object_sdk); + try { + await fs.promises.stat(bucket_temp_dir_path); + } catch (err) { + assert.strictEqual(err.code, 'ENOENT'); + assert.match(err.message, /.noobaa-nsfs_/); + } + try { + await fs.promises.stat(bucket_config_path); + } catch (err) { + assert.strictEqual(err.code, 'ENOENT'); + assert.equal(err.message, "ENOENT: no such file or directory, stat '/tmp/test_bucketspace_fs/config_root/buckets/buckettempdir.json'"); + } + await fs.promises.stat(path.join(new_buckets_path, param.name)); + }); }); mocha.describe('set_bucket_versioning', function() { mocha.before(async function() { diff --git a/src/test/unit_tests/test_nc_nsfs_health.js b/src/test/unit_tests/test_nc_nsfs_health.js index f99e4083ea..e14d491c64 100644 --- a/src/test/unit_tests/test_nc_nsfs_health.js +++ b/src/test/unit_tests/test_nc_nsfs_health.js @@ -237,7 +237,7 @@ mocha.describe('nsfs nc health', function() { Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - Health.config_root = config_root_invalid; + Health.config_root = config_root_invalid; const get_service_state = sinon.stub(Health, "get_service_state"); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); diff --git a/src/util/mongo_utils.js b/src/util/mongo_utils.js index 0065bb757f..96f1cc1bdc 100644 --- a/src/util/mongo_utils.js +++ b/src/util/mongo_utils.js @@ -162,6 +162,15 @@ function is_object_id(id) { return (id instanceof mongodb.ObjectId); } +function mongoObjectId() { + // eslint-disable-next-line no-bitwise + const timestamp = (new Date().getTime() / 1000 | 0).toString(16); + return timestamp + 'xxxxxxxxxxxxxxxx'.replace(/[x]/g, function() { + // eslint-disable-next-line no-bitwise + return (Math.random() * 16 | 0).toString(16); + }).toLowerCase(); +}; + // function is_err_duplicate_key(err) { // return err && err.code === 11000; // } @@ -240,3 +249,4 @@ exports.is_object_id = is_object_id; // exports.check_update_one = check_update_one; // exports.make_object_diff = make_object_diff; // exports.get_db_stats = get_db_stats; +exports.mongoObjectId = mongoObjectId diff --git a/src/util/native_fs_utils.js b/src/util/native_fs_utils.js index abaac41041..90b369c4fe 100644 --- a/src/util/native_fs_utils.js +++ b/src/util/native_fs_utils.js @@ -452,7 +452,13 @@ function validate_bucket_creation(params) { } } -async function config_file_exists(fs_context, config_path, use_lstat) { +/** + * Validate the path param exists or not + * @param {nb.NativeFSContext} fs_context + * @param {string} config_path + * @param {boolean} use_lstat + */ +async function is_path_exists(fs_context, config_path, use_lstat=false) { try { await nb_native().fs.stat(fs_context, config_path, { use_lstat }); } catch (err) { @@ -462,6 +468,31 @@ async function config_file_exists(fs_context, config_path, use_lstat) { return true; } +/** + * delete bucket specific temp folder from bucket storage path, config.NSFS_TEMP_DIR_NAME_ + * @param {string} dir + * @param {nb.NativeFSContext} fs_context + * @param {boolean} is_temp + */ +async function folder_delete(dir, fs_context, is_temp = false) { + const exists = await is_path_exists(fs_context, dir); + if (!exists && is_temp) { + return; + } + const entries = await nb_native().fs.readdir(fs_context, dir); + const results = await Promise.all(entries.map(entry => { + const fullPath = path.join(dir, entry.name); + const task = isDirectory(entry) ? folder_delete(fullPath, fs_context) : + nb_native().fs.unlink(fs_context, fullPath); + return task.catch(error => ({ error })); + })); + results.forEach(result => { + // Ignore missing files/directories; bail on other errors + if (result && result.error && result.error.code !== 'ENOENT') throw result.error; + }); + await nb_native().fs.rmdir(fs_context, dir); +} + exports.get_umasked_mode = get_umasked_mode; exports._make_path_dirs = _make_path_dirs; exports._create_path = _create_path; @@ -490,4 +521,5 @@ exports.update_config_file = update_config_file; exports.isDirectory = isDirectory; exports.get_process_fs_context = get_process_fs_context; exports.validate_bucket_creation = validate_bucket_creation; -exports.config_file_exists = config_file_exists; +exports.is_path_exists = is_path_exists; +exports.folder_delete = folder_delete;