Skip to content

Commit

Permalink
NSFS | NC | from-file in account
Browse files Browse the repository at this point in the history 
Signed-off-by: shirady <57721533+shirady@users.noreply.github.com>
  • Loading branch information
@shirady
shirady committed Feb 18, 2024
1 parent 563e0ab commit 3127b3e
Show file tree
Hide file tree
Showing 7 changed files with 324 additions and 88 deletions.
4 changes: 2 additions & 2 deletions docs/design/NonContainerizedNSFSDesign.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ node src/cmd/nsfs ../standalon/nsfs_root --config_dir ../standalon/fs_config

```json
{
"_id": "65cb1e7c9e6ae40d499c0ae3", // _id automatically generated
"name": "user1",
"email": "user1", // the email will be internally (the account name), email will not be set by user
"creation_date": "2024-01-11T08:24:14.937Z",
Expand All @@ -41,8 +42,7 @@ node src/cmd/nsfs ../standalon/nsfs_root --config_dir ../standalon/fs_config
"gid": 1001, //
"new_buckets_path": "/",
},
"allow_bucket_creation": true,
"_id": "65cb1e7c9e6ae40d499c0ae3" // _id automatically generated
"allow_bucket_creation": true
}
```

Expand Down
25 changes: 22 additions & 3 deletions docs/non_containerized_NSFS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,13 +467,32 @@ sudo node src/cmd/manage_nsfs bucket delete --config_root ../standalon/config_ro
sudo node src/cmd/manage_nsfs bucket list --config_root ../standalon/config_root 2>/dev/null
```
NSFS management CLI command will create both account and bucket dir if it's missing in the config_root path.
**Important**: All the Account/Bucket commands end with `2>/dev/null` to make sure there are no unwanted logs.


Using `from_file` flag:
- Users can also pass account or bucket values in JSON file (hereinafter referred to as "options JSON file") instead of passing them in CLI as arguments using flags.
- General use:
```
sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file /json_file/path
sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
```
NSFS management CLI command will create both account and bucket dir if it's missing in the config_root path.
- The options are key-value, where the key is the same as suggested flags. For example:
create JSON file for account:
```json
// JSON file of key-value options for creating an account
{
"name": "account-1001",
"uid": 1001,
"gid": 1001,
"new_buckets_path": "/tmp/nsfs_root1/my-bucket"
}
```
```
sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
```
- When using `from_file` flag the details about the account/bucket should be only inside the options JSON file.
- The JSON config file and JSON options file of account are different!

## NSFS Certificate

Expand Down
166 changes: 102 additions & 64 deletions src/cmd/manage_nsfs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ const manage_nsfs_glacier = require('../manage_nsfs/manage_nsfs_glacier');
const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils');
const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils');
const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE,
const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE,
LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS, GLACIER_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants');
const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;

Expand Down Expand Up @@ -99,11 +99,11 @@ async function main(argv = minimist(process.argv.slice(2))) {
config_root_backend = argv.config_root_backend ? String(argv.config_root_backend) : config.NSFS_NC_CONFIG_DIR_BACKEND;

await check_and_create_config_dirs();
const from_file = argv.from_file ? String(argv.from_file) : '';
const path_to_json_options = argv.from_file ? String(argv.from_file) : '';
if (type === TYPES.ACCOUNT) {
await account_management(argv, from_file);
await account_management(argv, path_to_json_options);
} else if (type === TYPES.BUCKET) {
await bucket_management(argv, from_file);
await bucket_management(argv, path_to_json_options);
} else if (type === TYPES.IP_WHITELIST) {
await whitelist_ips_management(argv);
} else if (type === TYPES.GLACIER) {
Expand Down Expand Up @@ -331,20 +331,25 @@ async function manage_bucket_operations(action, data, argv) {
}
}

async function account_management(argv, from_file) {
async function account_management(argv, path_to_json_options) {
const type = argv._[0] || '';
const action = argv._[1] || '';
const show_secrets = Boolean(argv.show_secrets) || false;
const data = await fetch_account_data(argv, from_file);
let user_input;
// currently we use from_file only in add action
if (path_to_json_options) user_input = await get_options_from_file(type, action, path_to_json_options);
user_input = user_input || argv;
const data = await fetch_account_data(action, user_input);
await manage_account_operations(action, data, show_secrets, argv);
}

/**
* set_access_keys will set the access keys either given as args or generated.
* @param {{ access_key: any; secret_key: any; }} argv
* set_access_keys will set the access keys either given or generated.
* @param {string} access_key
* @param {string} secret_key
* @param {boolean} generate a flag for generating the access_keys automatically
*/
function set_access_keys(argv, generate) {
const { access_key, secret_key } = argv;
function set_access_keys(access_key, secret_key, generate) {
let generated_access_key;
let generated_secret_key;
if (generate) {
Expand All @@ -360,57 +365,36 @@ function set_access_keys(argv, generate) {
}

// in name and new_name we allow type number, hence convert it to string
async function fetch_account_data(argv, from_file) {
let data;
let access_keys = [{
access_key: undefined,
secret_key: undefined,
}];
let new_access_key;
const action = argv._[1] || '';
if (from_file) {
const fs_context = native_fs_utils.get_process_fs_context();
const raw_data = (await nb_native().fs.readFile(fs_context, from_file)).data;
data = JSON.parse(raw_data.toString());
// GAP - from-file is not validated
}
if (action !== ACTIONS.LIST && action !== ACTIONS.STATUS) _validate_access_keys(argv);
if (action === ACTIONS.ADD || action === ACTIONS.STATUS) {
const regenerate = action === ACTIONS.ADD;
access_keys = set_access_keys(argv, regenerate);
} else if (action === ACTIONS.UPDATE) {
access_keys = set_access_keys(argv, Boolean(argv.regenerate));
new_access_key = access_keys[0].access_key;
access_keys[0].access_key = undefined; //Setting it as undefined so we can replace the symlink
}

if (!data) {
data = _.omitBy({
name: _.isUndefined(argv.name) ? undefined : String(argv.name),
email: _.isUndefined(argv.name) ? undefined : String(argv.name), // temp, keep the email internally
creation_date: action === ACTIONS.ADD ? new Date().toISOString() : undefined,
wide: argv.wide,
new_name: _.isUndefined(argv.new_name) ? undefined : String(argv.new_name),
new_access_key,
access_keys,
nsfs_account_config: {
distinguished_name: argv.user,
uid: argv.user ? undefined : argv.uid,
gid: argv.user ? undefined : argv.gid,
new_buckets_path: argv.new_buckets_path,
fs_backend: argv.fs_backend ? String(argv.fs_backend) : config.NSFS_NC_STORAGE_BACKEND
}
}, _.isUndefined);
}
async function fetch_account_data(action, user_input) {
const { access_keys, new_access_key } = get_access_keys(action, user_input);
let data = {
// added undefined values to keep the order the properties when printing the data object
_id: undefined,
name: _.isUndefined(user_input.name) ? undefined : String(user_input.name),
email: _.isUndefined(user_input.name) ? undefined : String(user_input.name), // temp, keep the email internally
creation_date: action === ACTIONS.ADD ? new Date().toISOString() : undefined,
wide: user_input.wide,
new_name: _.isUndefined(user_input.new_name) ? undefined : String(user_input.new_name),
new_access_key,
access_keys,
nsfs_account_config: {
distinguished_name: user_input.user,
uid: user_input.user ? undefined : user_input.uid,
gid: user_input.user ? undefined : user_input.gid,
new_buckets_path: user_input.new_buckets_path,
fs_backend: user_input.fs_backend ? String(user_input.fs_backend) : config.NSFS_NC_STORAGE_BACKEND
}
};
if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) {
// @ts-ignore
data = _.omitBy(data, _.isUndefined);
data = await fetch_existing_account_data(data);
}

// override values
// access_key as SensitiveString
data.access_keys[0].access_key = _.isUndefined(data.access_keys[0].access_key) ? undefined :
new SensitiveString(String(data.access_keys[0].access_key));
new SensitiveString(String(data.access_keys[0].access_key));
// secret_key as SensitiveString
data.access_keys[0].secret_key = _.isUndefined(data.access_keys[0].secret_key) ? undefined :
new SensitiveString(String(data.access_keys[0].secret_key));
Expand Down Expand Up @@ -690,6 +674,56 @@ async function get_config_data(config_file_path, show_secrets = false) {
return config_data;
}

/**
* get_options_from_file will read a JSON file that include key-value of the options
* (instead of flags) and return its content
* @param {string} type
* @param {string} action
* @param {string} file_path
*/
async function get_options_from_file(type, action, file_path) {
const fs_context = native_fs_utils.get_process_fs_context();
try {
const raw_data = (await nb_native().fs.readFile(fs_context, file_path)).data;
const input_options_with_data = JSON.parse(raw_data.toString());
const input_options = Object.keys(input_options_with_data);
if (input_options.includes(FROM_FILE)) {
const details = `${FROM_FILE} should not be passed inside json options`;
throw_cli_error(ManageCLIError.InvalidArgument, details);
}
validate_no_extra_options(type, action, input_options);
validate_options_type_by_value(input_options_with_data);
return input_options_with_data;
} catch (err) {
if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.InvalidFilePath, file_path);
if (err instanceof SyntaxError) throw_cli_error(ManageCLIError.InvalidJSONFile, file_path);
throw err;
}
}

/**
* get_access_keys will return the access_keys and new_access_key according to the user input
* and action
* @param {string} action
* @param {object} user_input
*/
function get_access_keys(action, user_input) {
let access_keys = [{
access_key: undefined,
secret_key: undefined,
}];
let new_access_key;
if (action !== ACTIONS.LIST && action !== ACTIONS.STATUS) _validate_access_keys(user_input.access_key, user_input.secret_key);
if (action === ACTIONS.ADD || action === ACTIONS.STATUS) {
const regenerate = action === ACTIONS.ADD;
access_keys = set_access_keys(user_input.access_key, user_input.secret_key, regenerate);
} else if (action === ACTIONS.UPDATE) {
access_keys = set_access_keys(user_input.access_key, user_input.secret_key, Boolean(user_input.regenerate));
new_access_key = access_keys[0].access_key;
access_keys[0].access_key = undefined; //Setting it as undefined so we can replace the symlink
}
return { access_keys, new_access_key };
}

///////////////////////////
/// VALIDATIONS ///
Expand Down Expand Up @@ -811,7 +845,7 @@ function validate_flags_arguments(type, action, argv) {
validate_no_extra_options(type, action, input_options);
const input_options_with_data = { ...argv };
delete input_options_with_data._;
validate_options_type_by_value(type, input_options_with_data);
validate_options_type_by_value(input_options_with_data);
}

/**
Expand Down Expand Up @@ -839,7 +873,11 @@ function validate_type_and_action(type, action) {
*/
function validate_no_extra_options(type, action, input_options) {
let valid_options; // for performance, we use Set as data structure
if (type === TYPES.BUCKET) {
const from_file_condition = (type === TYPES.ACCOUNT || type === TYPES.BUCKET) &&
action === ACTIONS.ADD && input_options.includes(FROM_FILE);
if (from_file_condition) {
valid_options = VALID_OPTIONS.from_file_options;
} else if (type === TYPES.BUCKET) {
valid_options = VALID_OPTIONS.bucket_options[action];
} else if (type === TYPES.ACCOUNT) {
valid_options = VALID_OPTIONS.account_options[action];
Expand All @@ -855,16 +893,16 @@ function validate_no_extra_options(type, action, input_options) {
`${invalid_input_options[0]} is an invalid option` :
`${invalid_input_options.join(', ')} are invalid options`;
const supported_option_msg = `Supported options are: ${[...valid_options].join(', ')}`;
const err_msg = `${invalid_option_msg} for ${type_and_action}. ${supported_option_msg}`;
throw_cli_error(ManageCLIError.InvalidArgument, err_msg);
let details = `${invalid_option_msg} for ${type_and_action}. ${supported_option_msg}`;
if (from_file_condition) details += ` (when using ${FROM_FILE} flag only partial list of flags are supported)`;
throw_cli_error(ManageCLIError.InvalidArgument, details);
}
}
/**
* validate_options_type_by_value check the type of the value that match what we expect.
* @param {string} type
* @param {object} input_options_with_data object with flag (key) and value
*/
function validate_options_type_by_value(type, input_options_with_data) {
function validate_options_type_by_value(input_options_with_data) {
for (const [option, value] of Object.entries(input_options_with_data)) {
const type_of_option = OPTION_TYPE[option];
const type_of_value = typeof value;
Expand Down Expand Up @@ -923,24 +961,24 @@ function verify_whitelist_ips(ips_to_validate) {
}
}

function _validate_access_keys(argv) {
function _validate_access_keys(access_key, secret_key) {
// using the access_key flag requires also using the secret_key flag
if (!_.isUndefined(argv.access_key) && _.isUndefined(argv.secret_key)) {
if (!_.isUndefined(access_key) && _.isUndefined(secret_key)) {
throw_cli_error(ManageCLIError.MissingAccountSecretKeyFlag);
}
if (!_.isUndefined(argv.secret_key) && _.isUndefined(argv.access_key)) {
if (!_.isUndefined(secret_key) && _.isUndefined(access_key)) {
throw_cli_error(ManageCLIError.MissingAccountAccessKeyFlag);
}
// checking the complexity of access_key
if (!_.isUndefined(argv.access_key) && !string_utils.validate_complexity(argv.access_key, {
if (!_.isUndefined(access_key) && !string_utils.validate_complexity(access_key, {
require_length: 20,
check_uppercase: true,
check_lowercase: false,
check_numbers: true,
check_symbols: false,
})) throw_cli_error(ManageCLIError.AccountAccessKeyFlagComplexity);
// checking the complexity of secret_key
if (!_.isUndefined(argv.secret_key) && !string_utils.validate_complexity(argv.secret_key, {
if (!_.isUndefined(secret_key) && !string_utils.validate_complexity(secret_key, {
require_length: 40,
check_uppercase: true,
check_lowercase: true,
Expand Down
12 changes: 12 additions & 0 deletions src/manage_nsfs/manage_nsfs_cli_errors.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,18 @@ ManageCLIError.InvalidSchema = Object.freeze({
http_code: 400,
});

ManageCLIError.InvalidFilePath = Object.freeze({
code: 'InvalidFilePath',
message: 'Invalid file path',
http_code: 400,
});

ManageCLIError.InvalidJSONFile = Object.freeze({
code: 'InvalidJSONFile',
message: 'Invalid JSON file',
http_code: 400,
});

//////////////////////////////
//// IP WHITE LIST ERRORS ////
//////////////////////////////
Expand Down
14 changes: 9 additions & 5 deletions src/manage_nsfs/manage_nsfs_constants.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,19 @@ const GLACIER_ACTIONS = {
};

const GLOBAL_CONFIG_ROOT = 'config_root';
const GLOBAL_CONFIG_OPTIONS = new Set(['from_file', GLOBAL_CONFIG_ROOT, 'config_root_backend']);
const FROM_FILE = 'from_file';

const VALID_OPTIONS_ACCOUNT = {
'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', ...GLOBAL_CONFIG_OPTIONS]),
'update': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'new_name', 'regenerate', ...GLOBAL_CONFIG_OPTIONS]),
'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', FROM_FILE, 'config_root_backend', GLOBAL_CONFIG_ROOT]),
'update': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'new_name', 'regenerate', 'config_root_backend', GLOBAL_CONFIG_ROOT]),
'delete': new Set(['name', GLOBAL_CONFIG_ROOT]),
'list': new Set(['wide', 'show_secrets', GLOBAL_CONFIG_ROOT, 'gid', 'uid', 'user', 'name', 'access_key']),
'status': new Set(['name', 'access_key', 'show_secrets', GLOBAL_CONFIG_ROOT]),
};

const VALID_OPTIONS_BUCKET = {
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', ...GLOBAL_CONFIG_OPTIONS]),
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', ...GLOBAL_CONFIG_OPTIONS]),
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', FROM_FILE, 'config_root_backend', GLOBAL_CONFIG_ROOT]),
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'config_root_backend', GLOBAL_CONFIG_ROOT]),
'delete': new Set(['name', GLOBAL_CONFIG_ROOT]),
'list': new Set(['wide', 'name', GLOBAL_CONFIG_ROOT]),
'status': new Set(['name', GLOBAL_CONFIG_ROOT]),
Expand All @@ -49,11 +49,14 @@ const VALID_OPTIONS_GLACIER = {

const VALID_OPTIONS_WHITELIST = new Set(['ips', GLOBAL_CONFIG_ROOT]);

const VALID_OPTIONS_FROM_FILE = new Set(['from_file', 'config_root_backend', GLOBAL_CONFIG_ROOT]);

const VALID_OPTIONS = {
account_options: VALID_OPTIONS_ACCOUNT,
bucket_options: VALID_OPTIONS_BUCKET,
glacier_options: VALID_OPTIONS_GLACIER,
whitelist_options: VALID_OPTIONS_WHITELIST,
from_file_options: VALID_OPTIONS_FROM_FILE,
};

const OPTION_TYPE = {
Expand Down Expand Up @@ -87,6 +90,7 @@ exports.ACTIONS = ACTIONS;
exports.GLACIER_ACTIONS = GLACIER_ACTIONS;
exports.VALID_OPTIONS = VALID_OPTIONS;
exports.OPTION_TYPE = OPTION_TYPE;
exports.FROM_FILE = FROM_FILE;

exports.LIST_ACCOUNT_FILTERS = LIST_ACCOUNT_FILTERS;
exports.LIST_BUCKET_FILTERS = LIST_BUCKET_FILTERS;
Loading

0 comments on commit 3127b3e

Please sign in to comment.