Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NSFS | NC | Allow one or more buckets to use the same dir path #7718

Merged
merged 1 commit into from
Jan 22, 2024
Merged

NSFS | NC | Allow one or more buckets to use the same dir path #7718

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/cmd/health.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ const fork_response_code = {
const health_errors_tyes = {
PERSISTENT: 'PERSISTENT',
TEMPORARY: 'TEMPORARY',
}
};

//suppress aws sdk related commands.
process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = '1';
Expand Down Expand Up @@ -180,7 +180,7 @@ class NSFSHealth {
}
}
});
} catch(err) {
} catch (err) {
console.log('Error while pinging endpoint host :' + HOSTNAME + ', port ' + this.https_port, err);
return {
response: fork_response_code.NOT_RUNNING.response_code,
Expand Down
29 changes: 16 additions & 13 deletions src/cmd/manage_nsfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const nb_native = require('../util/nb_native');
const cloud_utils = require('../util/cloud_utils');
const string_utils = require('../util/string_utils');
const native_fs_utils = require('../util/native_fs_utils');
const mongo_utils = require('../util/mongo_utils');
const SensitiveString = require('../util/sensitive_string');
const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse;
Expand Down Expand Up @@ -52,7 +53,7 @@ async function check_and_create_config_dirs() {
for (const dir_path of pre_req_dirs) {
try {
const fs_context = native_fs_utils.get_process_fs_context();
const dir_exists = await native_fs_utils.config_file_exists(fs_context, dir_path);
const dir_exists = await native_fs_utils.is_path_exists(fs_context, dir_path);
if (dir_exists) {
dbg.log1('nsfs.check_and_create_config_dirs: config dir exists:', dir_path);
} else {
Expand Down Expand Up @@ -142,7 +143,7 @@ async function fetch_bucket_data(argv, from_file) {
data.s3_policy = JSON.parse(argv.bucket_policy.toString());
}
}
if (action === ACTIONS.UPDATE) {
if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) {
naveenpaul1 marked this conversation as resolved.
Show resolved Hide resolved
data = _.omitBy(data, _.isUndefined);
data = await fetch_existing_bucket_data(data);
}
Expand Down Expand Up @@ -188,9 +189,9 @@ async function add_bucket(data) {
await verify_bucket_owner(data.bucket_owner.unwrap());
const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
const bucket_conf_path = get_config_file_path(buckets_dir_path, data.name);
const exists = await native_fs_utils.config_file_exists(fs_context, bucket_conf_path);
const exists = await native_fs_utils.is_path_exists(fs_context, bucket_conf_path);
if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name.unwrap());

data._id = mongo_utils.mongoObjectId();
const data_json = JSON.stringify(data);
// We take an object that was stringify
// (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
Expand Down Expand Up @@ -266,7 +267,7 @@ async function update_bucket(data) {
const cur_bucket_config_path = get_config_file_path(buckets_dir_path, cur_name.unwrap());
const new_bucket_config_path = get_config_file_path(buckets_dir_path, data.name.unwrap());

const exists = await native_fs_utils.config_file_exists(fs_context, new_bucket_config_path);
const exists = await native_fs_utils.is_path_exists(fs_context, new_bucket_config_path);
if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name.unwrap());

data = JSON.stringify(_.omit(data, ['new_name']));
Expand All @@ -281,13 +282,15 @@ async function update_bucket(data) {

async function delete_bucket(data) {
await validate_bucket_args(data, ACTIONS.DELETE);

const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
const bucket_config_path = get_config_file_path(buckets_dir_path, data.name);
try {
const bucket_temp_dir_path = path.join(data.path, config.NSFS_TEMP_DIR_NAME + "_" + data._id);
await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context, true);
await native_fs_utils.delete_config_file(fs_context, buckets_dir_path, bucket_config_path);
} catch (err) {
if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name);
throw err;
}
write_stdout_response(ManageCLIResponse.BucketDeleted);
}
Expand Down Expand Up @@ -440,14 +443,14 @@ async function add_account(data) {
const account_config_path = get_config_file_path(accounts_dir_path, data.name);
const account_config_access_key_path = get_symlink_config_file_path(access_keys_dir_path, access_key);

const name_exists = await native_fs_utils.config_file_exists(fs_context, account_config_path);
const access_key_exists = await native_fs_utils.config_file_exists(fs_context, account_config_access_key_path, true);
const name_exists = await native_fs_utils.is_path_exists(fs_context, account_config_path);
const access_key_exists = await native_fs_utils.is_path_exists(fs_context, account_config_access_key_path, true);

if (name_exists || access_key_exists) {
const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists;
throw_cli_error(err_code);
}

data._id = mongo_utils.mongoObjectId();
data = JSON.stringify(data);
// We take an object that was stringify
// (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
Expand Down Expand Up @@ -487,8 +490,8 @@ async function update_account(data) {
const new_account_config_path = get_config_file_path(accounts_dir_path, data.name.unwrap());
const cur_access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, cur_access_key.unwrap());
const new_access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key.unwrap());
const name_exists = update_name && await native_fs_utils.config_file_exists(fs_context, new_account_config_path);
const access_key_exists = update_access_key && await native_fs_utils.config_file_exists(fs_context, new_access_key_config_path, true);
const name_exists = update_name && await native_fs_utils.is_path_exists(fs_context, new_account_config_path);
const access_key_exists = update_access_key && await native_fs_utils.is_path_exists(fs_context, new_access_key_config_path, true);
if (name_exists || access_key_exists) {
const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists;
throw_cli_error(err_code);
Expand Down Expand Up @@ -673,7 +676,7 @@ async function validate_bucket_args(data, action) {
if (is_undefined(data.system_owner)) throw_cli_error(ManageCLIError.MissingBucketEmailFlag);
if (!data.path) throw_cli_error(ManageCLIError.MissingBucketPathFlag);
const fs_context = native_fs_utils.get_process_fs_context();
const exists = await native_fs_utils.config_file_exists(fs_context, data.path);
const exists = await native_fs_utils.is_path_exists(fs_context, data.path);
if (!exists) {
throw_cli_error(ManageCLIError.InvalidStoragePath, data.path);
}
Expand Down Expand Up @@ -741,7 +744,7 @@ async function validate_account_args(data, action) {
return;
}
const fs_context = native_fs_utils.get_process_fs_context();
const exists = await native_fs_utils.config_file_exists(fs_context, data.nsfs_account_config.new_buckets_path);
const exists = await native_fs_utils.is_path_exists(fs_context, data.nsfs_account_config.new_buckets_path);
if (!exists) {
throw_cli_error(ManageCLIError.InvalidAccountNewBucketsPath, data.nsfs_account_config.new_buckets_path);
}
Expand Down
7 changes: 7 additions & 0 deletions src/sdk/bucketspace_fs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const _ = require('lodash');
const util = require('util');
const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils');
const mongo_utils = require('../util/mongo_utils');

const KeysSemaphore = require('../util/keys_semaphore');
const native_fs_utils = require('../util/native_fs_utils');
Expand Down Expand Up @@ -344,6 +345,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {

new_bucket_defaults(account, { name, tag, lock_enabled, force_md5_etag }, create_uls, bucket_storage_path) {
return {
_id: mongo_utils.mongoObjectId(),
name,
tag: js_utils.default_value(tag, undefined),
owner_account: account._id,
naveenpaul1 marked this conversation as resolved.
Show resolved Hide resolved
Expand Down Expand Up @@ -378,10 +380,15 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
full_path: path.join(this.fs_root, namespace_bucket_config.write_resource.path) // includes write_resource.path + bucket name (s3 flow)
}, object_sdk);
} else if (namespace_bucket_config) {
// S3 Delete for NSFS Manage buckets
const list = await ns.list_objects({ ...params, limit: 1 }, object_sdk);
if (list && list.objects && list.objects.length > 0) {
throw new RpcError('NOT_EMPTY', 'underlying directory has files in it');
}
const bucket = await object_sdk.read_bucket_sdk_config_info(params.name)
const bucket_temp_dir_path = path.join(namespace_bucket_config.write_resource.path,
config.NSFS_TEMP_DIR_NAME + "_" + bucket._id);
await native_fs_utils.folder_delete(bucket_temp_dir_path, this.fs_context, true);
}
dbg.log1(`BucketSpaceFS: delete_fs_bucket ${bucket_path}`);
// delete bucket config json file
Expand Down
21 changes: 3 additions & 18 deletions src/sdk/namespace_fs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1702,7 +1702,7 @@ class NamespaceFS {

await target_file.close(fs_context);
target_file = null;
if (config.NSFS_REMOVE_PARTS_ON_COMPLETE) await this._folder_delete(params.mpu_path, fs_context);
if (config.NSFS_REMOVE_PARTS_ON_COMPLETE) await native_fs_utils.folder_delete(params.mpu_path, fs_context);
return upload_info;
} catch (err) {
dbg.error(err);
Expand Down Expand Up @@ -1733,7 +1733,7 @@ class NamespaceFS {
const fs_context = this.prepare_fs_context(object_sdk);
await this._load_multipart(params, fs_context);
dbg.log0('NamespaceFS: abort_object_upload', params.mpu_path);
await this._folder_delete(params.mpu_path, fs_context);
await native_fs_utils.folder_delete(params.mpu_path, fs_context);
}

///////////////////
Expand Down Expand Up @@ -2362,21 +2362,6 @@ class NamespaceFS {
}
}

async _folder_delete(dir, fs_context) {
const entries = await nb_native().fs.readdir(fs_context, dir);
const results = await Promise.all(entries.map(entry => {
const fullPath = path.join(dir, entry.name);
const task = native_fs_utils.isDirectory(entry) ? this._folder_delete(fullPath, fs_context) :
nb_native().fs.unlink(fs_context, fullPath);
return task.catch(error => ({ error }));
}));
results.forEach(result => {
// Ignore missing files/directories; bail on other errors
if (result && result.error && result.error.code !== 'ENOENT') throw result.error;
});
await nb_native().fs.rmdir(fs_context, dir);
}

async create_uls(params, object_sdk) {
const fs_context = this.prepare_fs_context(object_sdk);
dbg.log0('NamespaceFS: create_uls fs_context:', fs_context, 'new_dir_path: ', params.full_path);
Expand All @@ -2398,7 +2383,7 @@ class NamespaceFS {
throw new RpcError('NOT_EMPTY', 'underlying directory has files in it');
}

await this._folder_delete(params.full_path, fs_context);
await native_fs_utils.folder_delete(params.full_path, fs_context);
} catch (err) {
throw this._translate_object_error_codes(err);
}
Expand Down
4 changes: 4 additions & 0 deletions src/server/system_services/schemas/nsfs_account_schema.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ module.exports = {
$id: 'account_schema',
type: 'object',
required: [
'_id',
'name',
'email',
'access_keys',
Expand All @@ -13,6 +14,9 @@ module.exports = {
'allow_bucket_creation',
],
properties: {
_id: {
type: 'string',
},
name: {
type: 'string'
},
Expand Down
7 changes: 7 additions & 0 deletions src/server/system_services/schemas/nsfs_bucket_schema.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ module.exports = {
$id: 'bucket_schema',
type: 'object',
required: [
'_id',
'name',
'system_owner',
'bucket_owner',
Expand All @@ -14,6 +15,12 @@ module.exports = {
'creation_date',
],
properties: {
_id: {
type: 'string',
},
owner_account: {
shirady marked this conversation as resolved.
Show resolved Hide resolved
type: 'string',
},
name: {
type: 'string',
},
Expand Down
5 changes: 5 additions & 0 deletions src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ describe('manage nsfs cli account flow', () => {
const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs');
const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/');
const defaults = {
_id: 'account1',
type: 'account',
name: 'account1',
email: 'account1@noobaa.io',
Expand Down Expand Up @@ -151,6 +152,7 @@ describe('manage nsfs cli account flow', () => {
const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs1');
const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs1/');
const defaults = {
_id: 'account1',
type: 'account',
name: 'account1',
email: 'account1@noobaa.io',
Expand Down Expand Up @@ -255,6 +257,7 @@ describe('manage nsfs cli account flow', () => {
const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs1');
const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs1/');
const defaults = [{
_id: 'account1',
type: 'account',
name: 'account1',
email: 'account1@noobaa.io',
Expand All @@ -264,6 +267,7 @@ describe('manage nsfs cli account flow', () => {
access_key: 'GIGiFAnjaaE7OKD5N7hA',
secret_key: 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE',
}, {
_id: 'account1',
type: 'account',
name: 'account2',
email: 'account2@noobaa.io',
Expand All @@ -273,6 +277,7 @@ describe('manage nsfs cli account flow', () => {
access_key: 'BIBiFAnjaaE7OKD5N7hA',
secret_key: 'BIBYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE',
}, {
_id: 'account1',
type: 'account',
name: 'account3',
email: 'account3@noobaa.io',
Expand Down
20 changes: 19 additions & 1 deletion src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,23 @@ describe('schema validation NC NSFS account', () => {
assert_validation(account_data, reason, message);
});

it('account without _id', () => {
const account_data = get_account_data();
delete account_data._id;
const reason = 'Test should have failed because of missing required property ' +
'_id';
const message = "must have required property '_id'";
assert_validation(account_data, reason, message);
});

it('account with undefined _id', () => {
const account_data = get_account_data();
account_data._id = undefined;
const reason = 'Test should have failed because of missing required property ' +
'_id';
const message = "must have required property '_id'";
assert_validation(account_data, reason, message);
});
});

describe('account with wrong types', () => {
Expand Down Expand Up @@ -312,14 +329,14 @@ describe('schema validation NC NSFS account', () => {
const message = 'must be equal to one of the allowed values';
assert_validation(account_data, reason, message);
});

});

});


function get_account_data() {
const account_name = 'account1';
const id = '65a62e22ceae5e5f1a758aa9';
const account_email = 'account1@noobaa.io';
const access_key = 'GIGiFAnjaaE7OKD5N7hA';
const secret_key = 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE';
Expand All @@ -330,6 +347,7 @@ function get_account_data() {
};

const account_data = {
_id: id,
name: account_name,
email: account_email,
access_keys: [{
Expand Down
Loading