NSFS | NC | from-file in account

1. Add the from_file option in the account that would include validation. The JSON file will include options (as key-value structure) that are the same as the flags we suggest. 2. Change the structure inside the main so that eventually it doesn't matter whether the parameters are from the flags arguments (argv) or file. 3. Move get_access_keys to a separate function. 4. Fix indentation in fetch_account_data in the part of access_key as SensitiveString. 5. Remove an unused parameter from validate_options_type_by_value. 6. In native_fs_utils.js rename the function read_config_file to read_file. 7. Update json in doc so _id will appear first (as fixed in this PR). Signed-off-by: shirady <57721533+shirady@users.noreply.github.com>
noobaa · Mar 7, 2024 · 0796e79 · 0796e79
1 parent 8363cf7
commit 0796e79
Show file tree

Hide file tree

Showing 9 changed files with 374 additions and 121 deletions.
diff --git a/docs/design/NonContainerizedNSFSDesign.md b/docs/design/NonContainerizedNSFSDesign.md
@@ -27,6 +27,7 @@ node src/cmd/nsfs ../standalon/nsfs_root --config_dir ../standalon/fs_config
 
 ```json
 {
+  "_id": "65cb1e7c9e6ae40d499c0ae3", // _id automatically generated
   "name": "user1",
   "email": "user1", // the email will be internally (the account name), email will not be set by user
   "creation_date": "2024-01-11T08:24:14.937Z",
@@ -41,8 +42,7 @@ node src/cmd/nsfs ../standalon/nsfs_root --config_dir ../standalon/fs_config
     "gid": 1001,   // 
     "new_buckets_path": "/",
   },
-  "allow_bucket_creation": true,
-  "_id": "65cb1e7c9e6ae40d499c0ae3" // _id automatically generated
+  "allow_bucket_creation": true
 }
 ```
 

diff --git a/docs/non_containerized_NSFS.md b/docs/non_containerized_NSFS.md
@@ -468,13 +468,32 @@ sudo node src/cmd/manage_nsfs bucket delete --config_root ../standalon/config_ro
 sudo node src/cmd/manage_nsfs bucket list --config_root ../standalon/config_root 2>/dev/null
 
 ```
+NSFS management CLI run will create both accounts, access_keys, and buckets directories if they are missing under the config_root directory.
 **Important**:  All the Account/Bucket commands end with `2>/dev/null` to make sure there are no unwanted logs.
 
-
+Using `from_file` flag:
+- For account and bucket creation users can also pass account or bucket values in JSON file (hereinafter referred to as "options JSON file") instead of passing them in CLI as arguments using flags.
+- General use:
+```
+sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
+sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
+```
+- The options are key-value, where the key is the same as suggested flags. For example:
+create JSON file for account:
+```json
+// JSON file of key-value options for creating an account
+{
+    "name": "account-1001",
+    "uid": 1001,
+    "gid": 1001,
+    "new_buckets_path": "/tmp/nsfs_root1/my-bucket"
+}
+```
 ```
-sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file /json_file/path
+sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file <options_JSON_file_path>
 ```
-NSFS management CLI command will create both account and bucket dir if it's missing in the config_root path.
+- When using `from_file` flag the details about the account/bucket should be only inside the options JSON file.
+- The JSON config file and JSON options file of account are different! 
 
 ## NSFS Certificate
 

diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js
diff --git a/src/manage_nsfs/manage_nsfs_cli_errors.js b/src/manage_nsfs/manage_nsfs_cli_errors.js
@@ -95,6 +95,18 @@ ManageCLIError.InvalidSchema = Object.freeze({
     http_code: 400,
 });
 
+ManageCLIError.InvalidFilePath = Object.freeze({
+    code: 'InvalidFilePath',
+    message: 'Invalid file path',
+    http_code: 400,
+});
+
+ManageCLIError.InvalidJSONFile = Object.freeze({
+    code: 'InvalidJSONFile',
+    message: 'Invalid JSON file',
+    http_code: 400,
+});
+
 //////////////////////////////
 //// IP WHITE LIST ERRORS ////
 //////////////////////////////

diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js
@@ -29,22 +29,23 @@ const CONFIG_SUBDIRS = {
 };
 
 const GLOBAL_CONFIG_ROOT = 'config_root';
-const GLOBAL_CONFIG_OPTIONS = new Set(['from_file', GLOBAL_CONFIG_ROOT, 'config_root_backend']);
+const GLOBAL_CONFIG_OPTIONS = new Set([GLOBAL_CONFIG_ROOT, 'config_root_backend']);
+const FROM_FILE = 'from_file';
 
 const VALID_OPTIONS_ACCOUNT = {
-    'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', ...GLOBAL_CONFIG_OPTIONS]),
+    'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]),
     'update': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'new_name', 'regenerate', ...GLOBAL_CONFIG_OPTIONS]),
-    'delete': new Set(['name', GLOBAL_CONFIG_ROOT]),
-    'list': new Set(['wide', 'show_secrets', GLOBAL_CONFIG_ROOT, 'gid', 'uid', 'user', 'name', 'access_key']),
-    'status': new Set(['name', 'access_key', 'show_secrets', GLOBAL_CONFIG_ROOT]),
+    'delete': new Set(['name', ...GLOBAL_CONFIG_OPTIONS]),
+    'list': new Set(['wide', 'show_secrets', 'gid', 'uid', 'user', 'name', 'access_key', ...GLOBAL_CONFIG_OPTIONS]),
+    'status': new Set(['name', 'access_key', 'show_secrets', ...GLOBAL_CONFIG_OPTIONS]),
 };
 
 const VALID_OPTIONS_BUCKET = {
-    'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', ...GLOBAL_CONFIG_OPTIONS]),
+    'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]),
     'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', ...GLOBAL_CONFIG_OPTIONS]),
-    'delete': new Set(['name', GLOBAL_CONFIG_ROOT]),
-    'list': new Set(['wide', 'name', GLOBAL_CONFIG_ROOT]),
-    'status': new Set(['name', GLOBAL_CONFIG_ROOT]),
+    'delete': new Set(['name', ...GLOBAL_CONFIG_OPTIONS]),
+    'list': new Set(['wide', 'name', ...GLOBAL_CONFIG_OPTIONS]),
+    'status': new Set(['name', ...GLOBAL_CONFIG_OPTIONS]),
 };
 
 const VALID_OPTIONS_GLACIER = {
@@ -53,13 +54,16 @@ const VALID_OPTIONS_GLACIER = {
     'expiry': new Set([ GLOBAL_CONFIG_ROOT]),
 };
 
-const VALID_OPTIONS_WHITELIST = new Set(['ips', GLOBAL_CONFIG_ROOT]);
+const VALID_OPTIONS_WHITELIST = new Set(['ips', ...GLOBAL_CONFIG_OPTIONS]);
+
+const VALID_OPTIONS_FROM_FILE = new Set(['from_file', ...GLOBAL_CONFIG_OPTIONS]);
 
 const VALID_OPTIONS = {
     account_options: VALID_OPTIONS_ACCOUNT,
     bucket_options: VALID_OPTIONS_BUCKET,
     glacier_options: VALID_OPTIONS_GLACIER,
     whitelist_options: VALID_OPTIONS_WHITELIST,
+    from_file_options: VALID_OPTIONS_FROM_FILE,
 };
 
 const OPTION_TYPE = {
@@ -97,6 +101,7 @@ exports.GLACIER_ACTIONS = GLACIER_ACTIONS;
 exports.CONFIG_SUBDIRS = CONFIG_SUBDIRS;
 exports.VALID_OPTIONS = VALID_OPTIONS;
 exports.OPTION_TYPE = OPTION_TYPE;
+exports.FROM_FILE = FROM_FILE;
 exports.BOOLEAN_STRING_VALUES = BOOLEAN_STRING_VALUES;
 
 exports.LIST_ACCOUNT_FILTERS = LIST_ACCOUNT_FILTERS;

diff --git a/src/manage_nsfs/manage_nsfs_help_utils.js b/src/manage_nsfs/manage_nsfs_help_utils.js
@@ -60,13 +60,8 @@ Flags:
 `;
 
 const GLOBAL_CONFIG_ROOT_ALL_FLAG = `
---config_root <string>                                (optional)        Use configuration files path (default config.NSFS_NC_DEFAULT_CONF_DIR)
-`;
-
-const GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS = `
---from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
---config_root <string>                                (optional)        Use configuration files path (default config.NSFS_NC_DEFAULT_CONF_DIR)
---config_root_backend <none | GPFS | CEPH_FS | NFSv4> (optional)        Use the filesystem type in the configuration (default config.NSFS_NC_CONFIG_DIR_BACKEND)
+--config_root <string>                                    (optional)                        Use configuration files path (default config.NSFS_NC_DEFAULT_CONF_DIR)
+--config_root_backend <none | GPFS | CEPH_FS | NFSv4>     (optional)                        Use the filesystem type in the configuration (default config.NSFS_NC_CONFIG_DIR_BACKEND)
 `;
 
 const ACCOUNT_FLAGS_ADD = `
@@ -83,6 +78,7 @@ Flags:
 --secret_key <string>                                 (optional)        Set the secret key for the account (default is generated)
 --fs_backend <none | GPFS | CEPH_FS | NFSv4>          (optional)        Set the filesystem type of new_buckets_path (default config.NSFS_NC_STORAGE_BACKEND)
 --allow_bucket_creation <true | false>                (optional)        Set the account to explicitly allow or block bucket creation
+--from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
 `;
 
 const ACCOUNT_FLAGS_UPDATE = `
@@ -145,6 +141,7 @@ Flags:
 --path <string>                                                         Set the bucket path
 --bucket_policy <string>                              (optional)        Set the bucket policy, type is a string of valid JSON policy
 --fs_backend <none | GPFS | CEPH_FS | NFSv4>          (optional)        Set the filesystem type (default config.NSFS_NC_STORAGE_BACKEND)
+--from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
 `;
 
 const BUCKET_FLAGS_UPDATE = `
@@ -239,12 +236,10 @@ function print_usage(type, action) {
 function print_help_account(action) {
     switch (action) {
         case ACTIONS.ADD:
-            process.stdout.write(ACCOUNT_FLAGS_ADD.trimStart() +
-                GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart());
+            process.stdout.write(ACCOUNT_FLAGS_ADD.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart());
             break;
         case ACTIONS.UPDATE:
-            process.stdout.write(ACCOUNT_FLAGS_UPDATE.trimStart() +
-                GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart());
+            process.stdout.write(ACCOUNT_FLAGS_UPDATE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart());
             break;
         case ACTIONS.DELETE:
             process.stdout.write(ACCOUNT_FLAGS_DELETE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart());
@@ -268,12 +263,10 @@ function print_help_account(action) {
 function print_help_bucket(action) {
     switch (action) {
         case ACTIONS.ADD:
-            process.stdout.write(BUCKET_FLAGS_ADD.trimStart() +
-                GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart());
+            process.stdout.write(BUCKET_FLAGS_ADD.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart());
             break;
         case ACTIONS.UPDATE:
-            process.stdout.write(BUCKET_FLAGS_UPDATE.trimStart() +
-                GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart());
+            process.stdout.write(BUCKET_FLAGS_UPDATE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart());
             break;
         case ACTIONS.DELETE:
             process.stdout.write(BUCKET_FLAGS_DELETE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart());