diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index 7985609980..a3be96ddf0 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -13,6 +13,7 @@ const native_fs_utils = require('../util/native_fs_utils'); const SensitiveString = require('../util/sensitive_string'); const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; +const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils'); const TYPES = { ACCOUNT: 'account', @@ -40,6 +41,16 @@ function write_stdout_response(response_code, detail) { process.exit(0); } +const buckets_dir_name = '/buckets'; +const accounts_dir_name = '/accounts'; +const access_keys_dir_name = '/access_keys'; + +let config_root; +let accounts_dir_path; +let access_keys_dir_path; +let buckets_dir_path; +let config_root_backend; + const HELP = ` Help: @@ -115,6 +126,7 @@ Bucket Options: # required for add, update, and delete --name (default none) Set the name for the bucket. + --bucket_policy (default none) Set a bucket policy for the bucket, type is a string of valid JSON policy --new_name (default none) Set a new name for the bucket. --config_root (default config.NSFS_NC_DEFAULT_CONF_DIR) Configuration files path for Noobaa standalon NSFS. --wide (default none) Will print the list with details (same as status but for all buckets) @@ -130,16 +142,13 @@ function print_usage(options) { process.exit(1); } -const buckets_dir_name = '/buckets'; -const accounts_dir_name = '/accounts'; -const access_keys_dir_name = '/access_keys'; -async function check_and_create_config_dirs(config_root) { +async function check_and_create_config_dirs() { const pre_req_dirs = [ config_root, - path.join(config_root, buckets_dir_name), - path.join(config_root, accounts_dir_name), - path.join(config_root, access_keys_dir_name) + buckets_dir_path, + accounts_dir_path, + access_keys_dir_path ]; for (const dir_path of pre_req_dirs) { try { @@ -170,17 +179,22 @@ async function main(argv = minimist(process.argv.slice(2))) { print_white_list: type === TYPES.IPWHITELIST }); } - const config_root = argv.config_root ? String(argv.config_root) : config.NSFS_NC_CONF_DIR; + config_root = argv.config_root ? String(argv.config_root) : config.NSFS_NC_CONF_DIR; if (!config_root) throw_cli_error(ManageCLIError.MissingConfigDirPath); - await check_and_create_config_dirs(config_root); + accounts_dir_path = path.join(config_root, accounts_dir_name); + access_keys_dir_path = path.join(config_root, access_keys_dir_name); + buckets_dir_path = path.join(config_root, buckets_dir_name); + config_root_backend = argv.config_root_backend ? String(argv.config_root_backend) : config.NSFS_NC_CONFIG_DIR_BACKEND; + + await check_and_create_config_dirs(); const from_file = argv.from_file ? String(argv.from_file) : ''; if (type === TYPES.ACCOUNT) { - await account_management(argv, config_root, from_file); + await account_management(argv, from_file); } else if (type === TYPES.BUCKET) { - await bucket_management(argv, config_root, from_file); + await bucket_management(argv, from_file); } else if (type === TYPES.IPWHITELIST) { - await whitelist_ips_management(argv, config_root); + await whitelist_ips_management(argv); } else { throw_cli_error(ManageCLIError.InvalidConfigType); } @@ -193,14 +207,13 @@ async function main(argv = minimist(process.argv.slice(2))) { } } -async function bucket_management(argv, config_root, from_file) { +async function bucket_management(argv, from_file) { const action = argv._[1] || ''; - const config_root_backend = String(argv.config_root_backend) || config.NSFS_NC_CONFIG_DIR_BACKEND; - const data = await fetch_bucket_data(argv, config_root, from_file); - await manage_bucket_operations(action, data, config_root, config_root_backend); + const data = await fetch_bucket_data(argv, from_file); + await manage_bucket_operations(action, data); } -async function fetch_bucket_data(argv, config_root, from_file) { +async function fetch_bucket_data(argv, from_file) { const action = argv._[1] || ''; let data; if (from_file) { @@ -223,9 +236,18 @@ async function fetch_bucket_data(argv, config_root, from_file) { fs_backend: argv.fs_backend === undefined ? undefined : String(argv.fs_backend) }; } + + if (argv.bucket_policy !== undefined) { + // bucket_policy deletion speficied with empty string '' + if (argv.bucket_policy === '') { + data.s3_policy = ''; + } else { + data.s3_policy = JSON.parse(argv.bucket_policy.toString()); + } + } if (action === ACTIONS.UPDATE) { data = _.omitBy(data, _.isUndefined); - data = await fetch_existing_bucket_data(config_root, data); + data = await fetch_existing_bucket_data(data); } data = { @@ -236,15 +258,15 @@ async function fetch_bucket_data(argv, config_root, from_file) { // update bucket identifier new_name: data.new_name && new SensitiveString(String(data.new_name)) }; + return data; } -async function fetch_existing_bucket_data(config_root, target) { - const bucket_path = path.join(config_root, buckets_dir_name); +async function fetch_existing_bucket_data(target) { let source; try { - const full_bucket_config_path = get_config_file_path(bucket_path, target.name); - source = await get_config_data(full_bucket_config_path); + const bucket_config_path = get_config_file_path(buckets_dir_path, target.name); + source = await get_config_data(bucket_config_path); } catch (err) { throw_cli_error(ManageCLIError.NoSuchBucket, target.name); } @@ -260,7 +282,7 @@ function get_symlink_config_file_path(config_type_path, file_name) { return path.join(config_type_path, file_name + '.symlink'); } -async function add_bucket(data, buckets_dir_path, config_root_backend) { +async function add_bucket(data) { await validate_bucket_args(data, ACTIONS.ADD); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); const bucket_conf_path = get_config_file_path(buckets_dir_path, data.name); @@ -272,11 +294,11 @@ async function add_bucket(data, buckets_dir_path, config_root_backend) { write_stdout_response(ManageCLIResponse.BucketCreated, data_json); } -async function get_bucket_status(data, bucket_config_path, config_root_backend) { +async function get_bucket_status(data) { await validate_bucket_args(data, ACTIONS.STATUS); try { - const bucket_path = get_config_file_path(bucket_config_path, data.name); + const bucket_path = get_config_file_path(buckets_dir_path, data.name); const config_data = await get_config_data(bucket_path); write_stdout_response(ManageCLIResponse.BucketStatus, config_data); } catch (err) { @@ -285,7 +307,7 @@ async function get_bucket_status(data, bucket_config_path, config_root_backend) } } -async function update_bucket(data, bucket_config_path, config_root_backend) { +async function update_bucket(data) { await validate_bucket_args(data, ACTIONS.UPDATE); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); @@ -294,54 +316,53 @@ async function update_bucket(data, bucket_config_path, config_root_backend) { const update_name = data.new_name && cur_name && data.new_name.unwrap() !== cur_name.unwrap(); if (!update_name) { - const full_bucket_config_path = get_config_file_path(bucket_config_path, data.name); + const bucket_config_path = get_config_file_path(buckets_dir_path, data.name); data = JSON.stringify(data); - await native_fs_utils.update_config_file(fs_context, bucket_config_path, full_bucket_config_path, data); + await native_fs_utils.update_config_file(fs_context, buckets_dir_path, bucket_config_path, data); write_stdout_response(ManageCLIResponse.BucketUpdated, data); return; } data.name = data.new_name; - const cur_bucket_config_path = get_config_file_path(bucket_config_path, cur_name.unwrap()); - const new_bucket_config_path = get_config_file_path(bucket_config_path, data.name.unwrap()); + const cur_bucket_config_path = get_config_file_path(buckets_dir_path, cur_name.unwrap()); + const new_bucket_config_path = get_config_file_path(buckets_dir_path, data.name.unwrap()); const exists = await native_fs_utils.config_file_exists(fs_context, new_bucket_config_path); if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name.unwrap()); data = JSON.stringify(_.omit(data, ['new_name'])); - await native_fs_utils.create_config_file(fs_context, bucket_config_path, new_bucket_config_path, data); - await native_fs_utils.delete_config_file(fs_context, bucket_config_path, cur_bucket_config_path); + await native_fs_utils.create_config_file(fs_context, buckets_dir_path, new_bucket_config_path, data); + await native_fs_utils.delete_config_file(fs_context, buckets_dir_path, cur_bucket_config_path); write_stdout_response(ManageCLIResponse.BucketUpdated, data); } -async function delete_bucket(data, buckets_schema_path, config_root_backend) { +async function delete_bucket(data) { await validate_bucket_args(data, ACTIONS.DELETE); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); - const full_bucket_config_path = get_config_file_path(buckets_schema_path, data.name); + const bucket_config_path = get_config_file_path(buckets_dir_path, data.name); try { - await native_fs_utils.delete_config_file(fs_context, buckets_schema_path, full_bucket_config_path); + await native_fs_utils.delete_config_file(fs_context, buckets_dir_path, bucket_config_path); } catch (err) { if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name); } write_stdout_response(ManageCLIResponse.BucketDeleted); } -async function manage_bucket_operations(action, data, config_root, config_root_backend) { - const bucket_config_path = path.join(config_root, buckets_dir_name); +async function manage_bucket_operations(action, data) { if (action === ACTIONS.ADD) { - await add_bucket(data, bucket_config_path, config_root_backend); + await add_bucket(data); } else if (action === ACTIONS.STATUS) { - await get_bucket_status(data, bucket_config_path, config_root_backend); + await get_bucket_status(data); } else if (action === ACTIONS.UPDATE) { - await update_bucket(data, bucket_config_path, config_root_backend); + await update_bucket(data); } else if (action === ACTIONS.DELETE) { - await delete_bucket(data, bucket_config_path, config_root_backend); + await delete_bucket(data); } else if (action === ACTIONS.LIST) { - let buckets = await list_config_files(bucket_config_path); + let buckets = await list_config_files(buckets_dir_path); if (!data.wide) buckets = buckets.map(item => ({ name: item.name })); write_stdout_response(ManageCLIResponse.BucketList, buckets); } else { @@ -349,12 +370,11 @@ async function manage_bucket_operations(action, data, config_root, config_root_b } } -async function account_management(argv, config_root, from_file) { +async function account_management(argv, from_file) { const action = argv._[1] || ''; const show_secrets = Boolean(argv.show_secrets) || false; - const config_root_backend = String(argv.config_root_backend) || config.NSFS_NC_CONFIG_DIR_BACKEND; - const data = await fetch_account_data(argv, config_root, from_file); - await manage_account_operations(action, data, config_root, config_root_backend, show_secrets, argv); + const data = await fetch_account_data(argv, from_file); + await manage_account_operations(action, data, show_secrets, argv); } /** @@ -378,7 +398,7 @@ function set_access_keys(argv, generate) { }]; } -async function fetch_account_data(argv, config_root, from_file) { +async function fetch_account_data(argv, from_file) { let data; let generate_access_keys = true; const action = argv._[1] || ''; @@ -417,7 +437,7 @@ async function fetch_account_data(argv, config_root, from_file) { } if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) { data = _.omitBy(data, _.isUndefined); - data = await fetch_existing_account_data(config_root, data); + data = await fetch_existing_account_data(data); } data = { @@ -444,12 +464,12 @@ async function fetch_account_data(argv, config_root, from_file) { return data; } -async function fetch_existing_account_data(config_root, target) { +async function fetch_existing_account_data(target) { let source; try { const account_path = target.name ? - get_config_file_path(path.join(config_root, accounts_dir_name), target.name) : - get_symlink_config_file_path(path.join(config_root, access_keys_dir_name), target.access_keys[0].access_key); + get_config_file_path(accounts_dir_path, target.name) : + get_symlink_config_file_path(access_keys_dir_path, target.access_keys[0].access_key); source = await get_config_data(account_path, true); } catch (err) { dbg.log1('NSFS Manage command: Could not find account', target, err); @@ -464,16 +484,16 @@ async function fetch_existing_account_data(config_root, target) { } -async function add_account(data, accounts_path, access_keys_path, config_root_backend) { +async function add_account(data) { await validate_account_args(data, ACTIONS.ADD); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); const access_key = data.access_keys[0].access_key; - const full_account_config_path = get_config_file_path(accounts_path, data.name); - const full_account_config_access_key_path = get_symlink_config_file_path(access_keys_path, access_key); + const account_config_path = get_config_file_path(accounts_dir_path, data.name); + const account_config_access_key_path = get_symlink_config_file_path(access_keys_dir_path, access_key); - const name_exists = await native_fs_utils.config_file_exists(fs_context, full_account_config_path); - const access_key_exists = await native_fs_utils.config_file_exists(fs_context, full_account_config_access_key_path, true); + const name_exists = await native_fs_utils.config_file_exists(fs_context, account_config_path); + const access_key_exists = await native_fs_utils.config_file_exists(fs_context, account_config_access_key_path, true); if (name_exists || access_key_exists) { const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists; @@ -481,14 +501,14 @@ async function add_account(data, accounts_path, access_keys_path, config_root_ba } data = JSON.stringify(data); - await native_fs_utils.create_config_file(fs_context, accounts_path, full_account_config_path, data); - await native_fs_utils._create_path(access_keys_path, fs_context, config.BASE_MODE_CONFIG_DIR); - await nb_native().fs.symlink(fs_context, full_account_config_path, full_account_config_access_key_path); + await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, data); + await native_fs_utils._create_path(access_keys_dir_path, fs_context, config.BASE_MODE_CONFIG_DIR); + await nb_native().fs.symlink(fs_context, account_config_path, account_config_access_key_path); write_stdout_response(ManageCLIResponse.AccountCreated, data); } -async function update_account(data, accounts_path, access_keys_path, config_root_backend) { +async function update_account(data) { await validate_account_args(data, ACTIONS.UPDATE); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); @@ -498,19 +518,19 @@ async function update_account(data, accounts_path, access_keys_path, config_root const update_access_key = data.new_access_key && cur_access_key && data.new_access_key.unwrap() !== cur_access_key.unwrap(); if (!update_name && !update_access_key) { - const full_account_config_path = get_config_file_path(accounts_path, data.name); + const account_config_path = get_config_file_path(accounts_dir_path, data.name); data = JSON.stringify(data); - await native_fs_utils.update_config_file(fs_context, accounts_path, full_account_config_path, data); + await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, data); write_stdout_response(ManageCLIResponse.AccountUpdated, data); return; } const data_name = data.new_name || cur_name; data.name = data_name; data.access_keys[0].access_key = data.new_access_key || cur_access_key; - const cur_account_config_path = get_config_file_path(accounts_path, cur_name.unwrap()); - const new_account_config_path = get_config_file_path(accounts_path, data.name.unwrap()); - const cur_access_key_config_path = get_symlink_config_file_path(access_keys_path, cur_access_key.unwrap()); - const new_access_key_config_path = get_symlink_config_file_path(access_keys_path, data.access_keys[0].access_key.unwrap()); + const cur_account_config_path = get_config_file_path(accounts_dir_path, cur_name.unwrap()); + const new_account_config_path = get_config_file_path(accounts_dir_path, data.name.unwrap()); + const cur_access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, cur_access_key.unwrap()); + const new_access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key.unwrap()); const name_exists = update_name && await native_fs_utils.config_file_exists(fs_context, new_account_config_path); const access_key_exists = update_access_key && await native_fs_utils.config_file_exists(fs_context, new_access_key_config_path, true); if (name_exists || access_key_exists) { @@ -519,10 +539,10 @@ async function update_account(data, accounts_path, access_keys_path, config_root } data = JSON.stringify(_.omit(data, ['new_name', 'new_access_key'])); if (update_name) { - await native_fs_utils.create_config_file(fs_context, accounts_path, new_account_config_path, data); - await native_fs_utils.delete_config_file(fs_context, accounts_path, cur_account_config_path); + await native_fs_utils.create_config_file(fs_context, accounts_dir_path, new_account_config_path, data); + await native_fs_utils.delete_config_file(fs_context, accounts_dir_path, cur_account_config_path); } else if (update_access_key) { - await native_fs_utils.update_config_file(fs_context, accounts_path, cur_account_config_path, data); + await native_fs_utils.update_config_file(fs_context, accounts_dir_path, cur_account_config_path, data); } // TODO: safe_unlink can be better but the current impl causing ELOOP - Too many levels of symbolic links // need to find a better way for atomic unlinking of symbolic links @@ -532,26 +552,26 @@ async function update_account(data, accounts_path, access_keys_path, config_root write_stdout_response(ManageCLIResponse.AccountUpdated, data); } -async function delete_account(data, accounts_path, access_keys_path, config_root_backend) { +async function delete_account(data) { await validate_account_args(data, ACTIONS.DELETE); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); - const account_config_path = get_config_file_path(accounts_path, data.name); - const access_key_config_path = get_symlink_config_file_path(access_keys_path, data.access_keys[0].access_key.unwrap()); + const account_config_path = get_config_file_path(accounts_dir_path, data.name); + const access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key.unwrap()); - await native_fs_utils.delete_config_file(fs_context, accounts_path, account_config_path); + await native_fs_utils.delete_config_file(fs_context, accounts_dir_path, account_config_path); await nb_native().fs.unlink(fs_context, access_key_config_path); write_stdout_response(ManageCLIResponse.AccountDeleted); } -async function get_account_status(data, accounts_path, access_keys_path, show_secrets) { +async function get_account_status(data, show_secrets) { await validate_account_args(data, ACTIONS.STATUS); try { const account_path = is_undefined(data.name) ? - get_symlink_config_file_path(access_keys_path, data.access_keys[0].access_key) : - get_config_file_path(accounts_path, data.name); + get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key) : + get_config_file_path(accounts_dir_path, data.name); const config_data = await get_config_data(account_path, show_secrets); write_stdout_response(ManageCLIResponse.AccountStatus, config_data); } catch (err) { @@ -563,19 +583,17 @@ async function get_account_status(data, accounts_path, access_keys_path, show_se } } -async function manage_account_operations(action, data, config_root, config_root_backend, show_secrets, argv) { - const accounts_path = path.join(config_root, accounts_dir_name); - const access_keys_path = path.join(config_root, access_keys_dir_name); +async function manage_account_operations(action, data, show_secrets, argv) { if (action === ACTIONS.ADD) { - await add_account(data, accounts_path, access_keys_path, config_root_backend); + await add_account(data); } else if (action === ACTIONS.STATUS) { - await get_account_status(data, accounts_path, access_keys_path, show_secrets); + await get_account_status(data, show_secrets); } else if (action === ACTIONS.UPDATE) { - await update_account(data, accounts_path, access_keys_path, config_root_backend); + await update_account(data); } else if (action === ACTIONS.DELETE) { - await delete_account(data, accounts_path, access_keys_path, config_root_backend); + await delete_account(data); } else if (action === ACTIONS.LIST) { - let accounts = await list_config_files(accounts_path, show_secrets); + let accounts = await list_config_files(accounts_dir_path, show_secrets); accounts = filter_account_results(accounts, argv); if (!data.wide) accounts = accounts.map(item => ({ name: item.name })); write_stdout_response(ManageCLIResponse.AccountList, accounts); @@ -676,6 +694,23 @@ async function validate_bucket_args(data, action) { } // fs_backend='' used for deletion of the fs_backend property if (data.fs_backend !== undefined && data.fs_backend !== 'GPFS' && data.fs_backend !== '') throw_cli_error(ManageCLIError.InvalidFSBackend); + if (data.s3_policy) { + try { + await bucket_policy_utils.validate_s3_policy(data.s3_policy, data.name.unwrap(), + async principal => { + const account_config_path = get_config_file_path(accounts_dir_path, principal); + try { + await nb_native().fs.stat(native_fs_utils.get_process_fs_context(), account_config_path); + return true; + } catch (err) { + return false; + } + }); + } catch (err) { + dbg.error('validate_bucket_args invalid bucket policy err:', err); + throw_cli_error(ManageCLIError.MalformedPolicy, data.s3_policy); + } + } } } @@ -730,7 +765,7 @@ function is_undefined(value) { return false; } -async function whitelist_ips_management(args, config_root) { +async function whitelist_ips_management(args) { const ips = args.ips; validate_whitelist_arg(ips); diff --git a/src/cmd/nsfs.js b/src/cmd/nsfs.js index 7404538aa2..24fe413d7b 100644 --- a/src/cmd/nsfs.js +++ b/src/cmd/nsfs.js @@ -136,6 +136,7 @@ class NsfsObjectSDK extends ObjectSDK { bucketspace, stats: endpoint_stats_collector.instance(), }); + this.nsfs_config_root = nsfs_config_root; this.nsfs_fs_root = fs_root; this.nsfs_fs_config = fs_config; this.nsfs_account = account; diff --git a/src/endpoint/s3/s3_rest.js b/src/endpoint/s3/s3_rest.js index 07c5c414bc..7fbe672df2 100755 --- a/src/endpoint/s3/s3_rest.js +++ b/src/endpoint/s3/s3_rest.js @@ -241,9 +241,9 @@ async function authorize_request_policy(req) { if (is_owner) return; throw new S3Error(S3Error.AccessDenied); } - + const account_identifier = req.object_sdk.nsfs_config_root ? account.name.unwrap() : account.email.unwrap(); const permission = await s3_bucket_policy_utils.has_bucket_policy_permission( - s3_policy, account.email.unwrap(), method, arn_path, req); + s3_policy, account_identifier, method, arn_path, req); if (permission === "DENY") throw new S3Error(S3Error.AccessDenied); if (permission === "ALLOW" || is_owner) return; diff --git a/src/manage_nsfs/manage_nsfs_cli_errors.js b/src/manage_nsfs/manage_nsfs_cli_errors.js index e766bab5d5..04075f0f3a 100644 --- a/src/manage_nsfs/manage_nsfs_cli_errors.js +++ b/src/manage_nsfs/manage_nsfs_cli_errors.js @@ -282,6 +282,13 @@ ManageCLIError.InvalidFSBackend = Object.freeze({ http_code: 400, }); +ManageCLIError.MalformedPolicy = Object.freeze({ + code: 'MalformedPolicy', + message: 'Invalid bucket policy', + http_code: 400, +}); + + ManageCLIError.FS_ERRORS_TO_MANAGE = Object.freeze({ EACCES: ManageCLIError.AccessDenied, EPERM: ManageCLIError.AccessDenied, diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js index a4040596b5..dfffacfb40 100644 --- a/src/sdk/bucketspace_fs.js +++ b/src/sdk/bucketspace_fs.js @@ -16,6 +16,7 @@ const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils'); const { default: Ajv } = require('ajv'); const bucket_schema = require('../server/object_services/schemas/nsfs_bucket_schema'); const account_schema = require('../server/object_services/schemas/nsfs_account_schema'); +const { KEYWORDS } = require('../util/schema_keywords'); const common_api = require('../api/common_api'); const KeysSemaphore = require('../util/keys_semaphore'); @@ -27,6 +28,13 @@ const BUCKET_PATH = 'buckets'; const ACCOUNT_PATH = 'accounts'; const ACCESS_KEYS_PATH = 'access_keys'; const ajv = new Ajv({ verbose: true, allErrors: true }); +ajv.addKeyword(KEYWORDS.methods); +ajv.addKeyword(KEYWORDS.doc); +ajv.addKeyword(KEYWORDS.date); +ajv.addKeyword(KEYWORDS.idate); +ajv.addKeyword(KEYWORDS.objectid); +ajv.addKeyword(KEYWORDS.binary); +ajv.addKeyword(KEYWORDS.wrapper); ajv.addSchema(common_api); const bucket_semaphore = new KeysSemaphore(1); @@ -76,9 +84,9 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { } async _get_account_by_name(name) { - const access_key_config_path = this._get_account_config_path(name); + const account_config_path = this._get_account_config_path(name); try { - await nb_native().fs.stat(this.fs_context, access_key_config_path); + await nb_native().fs.stat(this.fs_context, account_config_path); return true; } catch (err) { return false; @@ -282,7 +290,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { async create_bucket(params, sdk) { return bucket_semaphore.surround_key(String(params.name), async () => { if (!sdk.requesting_account.allow_bucket_creation) { - throw new RpcError('UNAUTHORIZED', 'Not allowed to create new buckets') + throw new RpcError('UNAUTHORIZED', 'Not allowed to create new buckets'); } if (!sdk.requesting_account.nsfs_account_config || !sdk.requesting_account.nsfs_account_config.new_buckets_path) { throw new RpcError('MISSING_NSFS_ACCOUNT_CONFIGURATION'); diff --git a/src/server/object_services/schemas/nsfs_bucket_schema.js b/src/server/object_services/schemas/nsfs_bucket_schema.js index 14b86674ca..17d6560738 100644 --- a/src/server/object_services/schemas/nsfs_bucket_schema.js +++ b/src/server/object_services/schemas/nsfs_bucket_schema.js @@ -41,5 +41,8 @@ module.exports = { fs_backend: { $ref: 'common_api#/definitions/fs_backend' }, + s3_policy: { + $ref: 'common_api#/definitions/bucket_policy', + } } }; diff --git a/src/test/unit_tests/test_nc_nsfs_cli.js b/src/test/unit_tests/test_nc_nsfs_cli.js index 35ad67eab6..6f28852f96 100644 --- a/src/test/unit_tests/test_nc_nsfs_cli.js +++ b/src/test/unit_tests/test_nc_nsfs_cli.js @@ -13,6 +13,7 @@ const nb_native = require('../../util/nb_native'); const config_module = require('../../../config'); const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); const { ManageCLIResponse } = require('../../manage_nsfs/manage_nsfs_cli_responses'); +const test_utils = require('../system_tests/test_utils'); const MAC_PLATFORM = 'darwin'; let tmp_fs_path = '/tmp/test_bucketspace_fs'; @@ -64,14 +65,39 @@ mocha.describe('manage_nsfs cli', function() { const bucket_on_gpfs = 'bucketgpfs1'; const owner_email = 'user1@noobaa.io'; const bucket_path = `${root_path}${name}/`; + const bucket_with_policy = 'bucket-with-policy'; + const bucket_policy = test_utils.generate_s3_policy('*', bucket_with_policy, ['s3:*']).policy; + const bucket1_policy = test_utils.generate_s3_policy('*', name, ['s3:*']).policy; + const invalid_bucket_policy = test_utils.generate_s3_policy('invalid_account', name, ['s3:*']).policy; + const empty_bucket_policy = ''; + const schema_dir = 'buckets'; let bucket_options = { config_root, name, owner_email, bucket_path }; const gpfs_bucket_options = { config_root, name: bucket_on_gpfs, owner_email, bucket_path, fs_backend: 'GPFS' }; + const bucket_with_policy_options = { ...bucket_options, bucket_policy: bucket_policy, name: bucket_with_policy }; + + mocha.it('cli bucket create with invalid bucket policy - should fail', async function() { + const action = nc_nsfs_manage_actions.ADD; + try { + await fs_utils.create_fresh_path(bucket_path); + await fs_utils.file_must_exist(bucket_path); + await exec_manage_cli(type, action, { ...bucket_options, bucket_policy: invalid_bucket_policy}); + assert.fail('should have failed with invalid bucket policy'); + } catch (err) { + assert_error(err, ManageCLIError.MalformedPolicy); + } + }); + + mocha.it('cli bucket create - bucket_with_policy', async function() { + const action = nc_nsfs_manage_actions.ADD; + await exec_manage_cli(type, action, bucket_with_policy_options); + const bucket = await read_config_file(config_root, schema_dir, bucket_with_policy); + assert_bucket(bucket, bucket_with_policy_options); + await assert_config_file_permissions(config_root, schema_dir, bucket_with_policy); + }); mocha.it('cli bucket create', async function() { const action = nc_nsfs_manage_actions.ADD; - await fs_utils.create_fresh_path(bucket_path); - await fs_utils.file_must_exist(bucket_path); const bucket_status = await exec_manage_cli(type, action, bucket_options); assert_response(action, type, bucket_status, bucket_options); const bucket = await read_config_file(config_root, schema_dir, name); @@ -100,14 +126,14 @@ mocha.describe('manage_nsfs cli', function() { mocha.it('cli bucket list', async function() { const action = nc_nsfs_manage_actions.LIST; const bucket_list = await exec_manage_cli(type, action, { config_root }); - const expected_list = [{ name }]; + const expected_list = [{ name }, { name: bucket_with_policy }]; assert_response(action, type, bucket_list, expected_list); }); mocha.it('cli bucket list - wide', async function() { const action = nc_nsfs_manage_actions.LIST; const bucket_list = await exec_manage_cli(type, action, { config_root, wide: true }); - const expected_list = [bucket_options]; + const expected_list = [bucket_options, bucket_with_policy_options]; assert_response(action, type, bucket_list, expected_list, undefined, true); }); @@ -151,6 +177,35 @@ mocha.describe('manage_nsfs cli', function() { await assert_config_file_permissions(config_root, schema_dir, name); }); + mocha.it('cli bucket update invalid bucket policy - should fail', async function() { + const action = nc_nsfs_manage_actions.UPDATE; + const update_options = { config_root, bucket_policy: invalid_bucket_policy, name }; + try { + await exec_manage_cli(type, action, update_options); + assert.fail('should have failed with invalid bucket policy'); + } catch (err) { + assert_error(err, ManageCLIError.MalformedPolicy); + } + }); + + mocha.it('cli bucket update bucket policy', async function() { + const action = nc_nsfs_manage_actions.UPDATE; + bucket_options = { ...bucket_options, bucket_policy: bucket1_policy }; + await exec_manage_cli(type, action, bucket_options); + const bucket = await read_config_file(config_root, schema_dir, bucket_options.name); + assert_bucket(bucket, bucket_options); + await assert_config_file_permissions(config_root, schema_dir, bucket_options.name); + }); + + mocha.it('cli bucket update bucket policy - delete bucket policy', async function() { + const action = nc_nsfs_manage_actions.UPDATE; + bucket_options = { ...bucket_options, bucket_policy: empty_bucket_policy }; + await exec_manage_cli(type, action, bucket_options); + const bucket = await read_config_file(config_root, schema_dir, bucket_options.name); + assert_bucket(bucket, bucket_options); + await assert_config_file_permissions(config_root, schema_dir, bucket_options.name); + }); + mocha.it('cli bucket update bucket name', async function() { const action = nc_nsfs_manage_actions.UPDATE; const update_options = { config_root, new_name: 'bucket2', name }; @@ -748,7 +803,7 @@ async function assert_config_file_permissions(config_root, schema_dir, config_fi const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? '.symlink' : '.json')); const { stat } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, config_path); // 33152 means 600 (only owner has read and write permissions) - assert.ok(stat.mode, 33152); + assert.equal(stat.mode, 33152); } function assert_error(err, expect_error) { @@ -769,12 +824,16 @@ function assert_response(action, type, actual_res, expected_res, show_secrets, w } else if (action === nc_nsfs_manage_actions.DELETE) { assert.equal(parsed.response.code, ManageCLIResponse.BucketDeleted.code); } else if (action === nc_nsfs_manage_actions.LIST) { - if (wide) { - for (let i = 0; i < parsed.response.reply.length; i++) { - assert_bucket(parsed.response.reply[i], expected_res[i]); + assert.equal(parsed.response.reply.length, expected_res.length); + for (let i = 0; i < parsed.response.reply.length; i++) { + const name = parsed.response.reply[i].name; + const expected_res_by_name = expected_res.find(expected => expected.name === name); + if (wide) { + assert_bucket(parsed.response.reply[i], expected_res_by_name); + } else { + assert.deepEqual(parsed.response.reply[i], expected_res_by_name); + } - } else { - assert.deepEqual(parsed.response.reply, expected_res); } } else { assert.fail(`Invalid command action - ${action}`); @@ -810,6 +869,7 @@ function assert_bucket(bucket, bucket_options) { assert.strictEqual(bucket.should_create_underlying_storage, false); assert.strictEqual(bucket.versioning, 'DISABLED'); assert.strictEqual(bucket.fs_backend, bucket_options.fs_backend); + assert.deepStrictEqual(bucket.s3_policy, bucket_options.bucket_policy); return true; } @@ -842,6 +902,7 @@ async function exec_manage_cli(type, action, options) { const bucket_flags = (options.name ? `--name ${options.name}` : ``) + (options.owner_email ? ` --email ${options.owner_email}` : ``) + (options.fs_backend === undefined ? `` : ` --fs_backend '${options.fs_backend}'`) + + (options.bucket_policy === undefined ? `` : ` --bucket_policy '${JSON.stringify(options.bucket_policy)}'`) + (options.bucket_path ? ` --path ${options.bucket_path}` : ``); const account_flags = (options.name ? ` --name ${options.name}` : ``) +