NSFS | NC | from_file in bucket add

Signed-off-by: shirady <57721533+shirady@users.noreply.github.com>
noobaa · Mar 10, 2024 · 3729825 · 3729825
1 parent d1bf8c4
commit 3729825
Show file tree

Hide file tree

Showing 4 changed files with 286 additions and 68 deletions.
diff --git a/docs/non_containerized_NSFS.md b/docs/non_containerized_NSFS.md
@@ -474,24 +474,43 @@ NSFS management CLI run will create both accounts, access_keys, and buckets dire
 Using `from_file` flag:
 - For account and bucket creation users can also pass account or bucket values in JSON file (hereinafter referred to as "options JSON file") instead of passing them in CLI as arguments using flags.
 - General use:
-```
+
+```bash
 sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
 sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
 ```
-- The options are key-value, where the key is the same as suggested flags. For example:
-create JSON file for account:
+
+- The options are key-value, where the key is the same as suggested flags, for example:
+
+(1) Create JSON file for account
+
 ```json
-// JSON file of key-value options for creating an account
 {
     "name": "account-1001",
     "uid": 1001,
     "gid": 1001,
-    "new_buckets_path": "/tmp/nsfs_root1/my-bucket"
+    "new_buckets_path": "/tmp/nsfs_root1"
 }
 ```
+
+```bash
+sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file <options_account_JSON_file_path>
 ```
-sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
+
+(2) Create JSON file for bucket:
+
+```json
+{
+    "name": "account-1001-bucket-1",
+    "owner": "account-1001",
+    "path": "/tmp/nsfs_root1/account-1001-bucket-1"
+}
 ```
+
+```
+sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file <options_bucket_JSON_file_path>
+```
+
 - When using `from_file` flag the details about the account/bucket should be only inside the options JSON file.
 - The JSON config file and JSON options file of account are different! 
 

diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js
@@ -103,9 +103,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
         if (type === TYPES.ACCOUNT) {
             await account_management(action, user_input);
         } else if (type === TYPES.BUCKET) {
-            // GAP - from-file in bucket
-            const path_to_json_options = argv.from_file ? String(argv.from_file) : '';
-            await bucket_management(argv, path_to_json_options);
+            await bucket_management(action, user_input);
         } else if (type === TYPES.IP_WHITELIST) {
             await whitelist_ips_management(argv);
         } else if (type === TYPES.GLACIER) {
@@ -124,50 +122,44 @@ async function main(argv = minimist(process.argv.slice(2))) {
     }
 }
 
-async function bucket_management(argv, from_file) {
-    const action = argv._[1] || '';
-    const data = await fetch_bucket_data(argv, from_file);
-    await manage_bucket_operations(action, data, argv);
+async function bucket_management(action, user_input) {
+    const data = await fetch_bucket_data(action, user_input);
+    await manage_bucket_operations(action, data, user_input);
 }
 
 // in name and new_name we allow type number, hence convert it to string
-async function fetch_bucket_data(argv, from_file) {
-    const action = argv._[1] || '';
-    let data;
-    if (from_file) {
-        const fs_context = native_fs_utils.get_process_fs_context();
-        const raw_data = (await nb_native().fs.readFile(fs_context, from_file)).data;
-        data = JSON.parse(raw_data.toString());
-        // GAP - from-file is not validated
-    }
-    if (!data) {
+async function fetch_bucket_data(action, user_input) {
+    let data = {
         // added undefined values to keep the order the properties when printing the data object
-        data = {
-            _id: undefined,
-            name: _.isUndefined(argv.name) ? undefined : String(argv.name),
-            owner_account: undefined,
-            system_owner: argv.owner, // GAP - needs to be the system_owner (currently it is the account name)
-            bucket_owner: argv.owner,
-            wide: _.isUndefined(argv.wide) ? undefined : get_boolean_or_string_value(argv.wide),
-            tag: undefined, // if we would add the option to tag a bucket using CLI, this should be changed
-            versioning: action === ACTIONS.ADD ? 'DISABLED' : undefined,
-            creation_date: action === ACTIONS.ADD ? new Date().toISOString() : undefined,
-            path: argv.path,
-            should_create_underlying_storage: action === ACTIONS.ADD ? false : undefined,
-            new_name: _.isUndefined(argv.new_name) ? undefined : String(argv.new_name),
-            fs_backend: _.isUndefined(argv.fs_backend) ? config.NSFS_NC_STORAGE_BACKEND : String(argv.fs_backend)
+        _id: undefined,
+        name: _.isUndefined(user_input.name) ? undefined : String(user_input.name),
+        owner_account: undefined,
+        system_owner: user_input.owner, // GAP - needs to be the system_owner (currently it is the account name)
+        bucket_owner: user_input.owner,
+        wide: _.isUndefined(user_input.wide) ? undefined : get_boolean_or_string_value(user_input.wide),
+        tag: undefined, // if we would add the option to tag a bucket using CLI, this should be changed
+        versioning: action === ACTIONS.ADD ? 'DISABLED' : undefined,
+        creation_date: action === ACTIONS.ADD ? new Date().toISOString() : undefined,
+        path: user_input.path,
+        should_create_underlying_storage: action === ACTIONS.ADD ? false : undefined,
+        new_name: _.isUndefined(user_input.new_name) ? undefined : String(user_input.new_name),
+        fs_backend: _.isUndefined(user_input.fs_backend) ? config.NSFS_NC_STORAGE_BACKEND : String(user_input.fs_backend)
         };
-    }
 
-    if (argv.bucket_policy !== undefined) {
-        // bucket_policy deletion speficied with empty string ''
-        if (argv.bucket_policy === '') {
-            data.s3_policy = '';
-        } else {
-            data.s3_policy = JSON.parse(argv.bucket_policy.toString());
+    if (user_input.bucket_policy !== undefined) {
+        if (typeof user_input.bucket_policy === 'string') {
+            // bucket_policy deletion specified with empty string ''
+            if (user_input.bucket_policy === '') {
+                data.s3_policy = '';
+            } else {
+                data.s3_policy = JSON.parse(user_input.bucket_policy.toString());
+            }
+        } else { // it is object type
+            data.s3_policy = user_input.bucket_policy;
         }
     }
     if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) {
+        // @ts-ignore
         data = _.omitBy(data, _.isUndefined);
         data = await fetch_existing_bucket_data(data);
     }
@@ -316,7 +308,7 @@ async function delete_bucket(data) {
     write_stdout_response(ManageCLIResponse.BucketDeleted, '', {bucket: data.name});
 }
 
-async function manage_bucket_operations(action, data, argv) {
+async function manage_bucket_operations(action, data, user_input) {
     if (action === ACTIONS.ADD) {
         await add_bucket(data);
     } else if (action === ACTIONS.STATUS) {
@@ -326,7 +318,7 @@ async function manage_bucket_operations(action, data, argv) {
     } else if (action === ACTIONS.DELETE) {
         await delete_bucket(data);
     } else if (action === ACTIONS.LIST) {
-        const bucket_filters = _.pick(argv, LIST_BUCKET_FILTERS);
+        const bucket_filters = _.pick(user_input, LIST_BUCKET_FILTERS);
         const buckets = await list_config_files(TYPES.BUCKET, buckets_dir_path, data.wide, undefined, bucket_filters);
         write_stdout_response(ManageCLIResponse.BucketList, buckets);
     } else {
@@ -847,7 +839,7 @@ async function validate_input_types(type, action, argv) {
 
     // currently we use from_file only in add action
     const path_to_json_options = argv.from_file ? String(argv.from_file) : '';
-    if (type === TYPES.ACCOUNT && action === ACTIONS.ADD && path_to_json_options) {
+    if ((type === TYPES.ACCOUNT || type === TYPES.BUCKET) && action === ACTIONS.ADD && path_to_json_options) {
         const input_options_with_data_from_file = await get_options_from_file(path_to_json_options);
         const input_options_from_file = Object.keys(input_options_with_data_from_file);
         if (input_options_from_file.includes(FROM_FILE)) {
@@ -894,16 +886,18 @@ function validate_no_extra_options(type, action, input_options, is_options_from_
         valid_options = VALID_OPTIONS.bucket_options[action];
     } else if (type === TYPES.ACCOUNT) {
         valid_options = VALID_OPTIONS.account_options[action];
-        if (is_options_from_file) {
-            valid_options.delete('from_file');
-            valid_options.delete('config_root');
-            valid_options.delete('config_root_backend');
-        }
     } else if (type === TYPES.GLACIER) {
         valid_options = VALID_OPTIONS.glacier_options[action];
     } else {
         valid_options = VALID_OPTIONS.whitelist_options;
     }
+
+    if (is_options_from_file) {
+        valid_options.delete('from_file');
+        valid_options.delete('config_root');
+        valid_options.delete('config_root_backend');
+    }
+
     const invalid_input_options = input_options.filter(element => !valid_options.has(element));
     if (invalid_input_options.length > 0) {
         const type_and_action = type === TYPES.IP_WHITELIST ? type : `${type} ${action}`;
@@ -933,6 +927,10 @@ function validate_options_type_by_value(input_options_with_data) {
             if (['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets'].includes(option) && validate_boolean_string_value(value)) {
                 continue;
             }
+            // special case for bucket_policy (from_file)
+            if (option === 'bucket_policy' && type_of_value === 'object') {
+                continue;
+            }
             const details = `type of flag ${option} should be ${type_of_option}`;
             throw_cli_error(ManageCLIError.InvalidArgumentType, details);
         }

diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js
@@ -970,7 +970,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account and check the details
+            // create the account and check the details
             await exec_manage_cli(type, action, command_flags);
             // compare the details
             const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name);
@@ -984,7 +984,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account and check the details
+            // create the account and check the details
             await exec_manage_cli(type, action, command_flags);
             // compare the details
             const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name);
@@ -1002,7 +1002,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account and check the details
+            // create the account
             const res = await exec_manage_cli(type, action, command_flags);
             expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.AccountAccessKeyFlagComplexity.code);
         });
@@ -1014,7 +1014,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name, name }; // name should be in file only
-            // create tha account and check the details
+            // create the account
             const res = await exec_manage_cli(type, action, command_flags);
             expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code);
         });
@@ -1026,7 +1026,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account and check the details
+            // create the account
             const res = await exec_manage_cli(type, action, command_flags);
             expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code);
         });
@@ -1038,7 +1038,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account and check the details
+            // create the account
             const res = await exec_manage_cli(type, action, command_flags);
             expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code);
         });
@@ -1050,7 +1050,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account and check the details
+            // create the account
             const res = await exec_manage_cli(type, action, command_flags);
             expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code);
         });
@@ -1062,7 +1062,7 @@ describe('manage nsfs cli account flow', () => {
             // write the json_file_options
             const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options);
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account and check the details
+            // create the account
             const res = await exec_manage_cli(type, action, command_flags);
             expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgumentType.code);
         });
@@ -1085,7 +1085,7 @@ describe('manage nsfs cli account flow', () => {
             await fs.promises.writeFile(path_to_option_json_file_name, content);
             // write the json_file_options
             const command_flags = {config_root, from_file: path_to_option_json_file_name};
-            // create tha account
+            // create the account
             await exec_manage_cli(type, action, command_flags);
             // compare the details
             const res = await exec_manage_cli(type, action, command_flags);