Generate user cli doc

cli/src/alluxio.org/cli/cmd/cache/format.go

@@ -38,8 +38,14 @@ func (c *FormatCommand) Base() *env.BaseJavaCommand {
 func (c *FormatCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   Format.CommandName,
-		Short: "Format Alluxio worker nodes.",
-		Args:  cobra.NoArgs,
+		Short: "Format Alluxio worker running locally",
+		Long: `The format command formats the Alluxio worker on this host.
+This deletes all the cached data stored by the worker. Data in the under storage will not be changed.
+
+> Warning: Format should only be called when the worker is not running`,
+		Example: `# Format worker
+$ ./bin/alluxio cache format`,
+		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/cache/free.go

@@ -30,7 +30,6 @@ type FreeCommand struct {
 	*env.BaseJavaCommand
 	worker string
 	path   string
-	force  bool
 }
 
 func (c *FreeCommand) Base() *env.BaseJavaCommand {
@@ -39,18 +38,20 @@ func (c *FreeCommand) Base() *env.BaseJavaCommand {
 
 func (c *FreeCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
-		Use: Free.CommandName,
-		Short: "Synchronously free all blocks and directories of specific worker, " +
-			"or free the space occupied by a file or a directory in Alluxio",
+		Use:   Free.CommandName,
+		Short: "Synchronously free cached files along a path or held by a specific worker",
+		Example: `# Free a file by its path
+$ ./bin/alluxio cache free --path /path/to/file
+
+# Free files on a worker
+$ ./bin/alluxio cache free --worker <workerHostName>`,
 		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},
 	})
 	cmd.Flags().StringVar(&c.worker, "worker", "", "The worker to free")
 	cmd.Flags().StringVar(&c.path, "path", "", "The file or directory to free")
-	cmd.Flags().BoolVarP(&c.force, "force", "f", false,
-		"Force freeing pinned files in the directory")
 	cmd.MarkFlagsMutuallyExclusive("worker", "path")
 	return cmd
 }
@@ -60,11 +61,7 @@ func (c *FreeCommand) Run(args []string) error {
 	if c.worker == "" {
 		if c.path != "" {
 			// free directory
-			javaArgs = append(javaArgs, "free")
-			if c.force {
-				javaArgs = append(javaArgs, "-f")
-			}
-			javaArgs = append(javaArgs, c.path)
+			javaArgs = append(javaArgs, "free", c.path)
 		} else {
 			return stacktrace.NewError("neither worker nor path to free specified")
 		}

cli/src/alluxio.org/cli/cmd/conf/get.go

@@ -46,7 +46,29 @@ func (c *GetCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   fmt.Sprintf("%v [key]", Get.CommandName),
 		Short: "Look up a configuration value by its property key or print all configuration if no key is provided",
-		Args:  cobra.MaximumNArgs(1),
+		Long: `The get command prints the configured value for the given key.
+If the key is invalid, it returns a nonzero exit code.
+If the key is valid but isn't set, an empty string is printed.
+If no key is specified, the full configuration is printed.
+
+> Note: This command does not require the Alluxio cluster to be running.`,
+		Example: `# Display all the current node configuration
+$ ./bin/alluxio conf get
+
+# Display the value of a property key
+$ ./bin/alluxio conf get alluxio.master.hostname
+
+# Display the configuration of the current running Alluxio leading master
+$ ./bin/alluxio conf get --master
+
+# Display the source of the configuration
+$ ./bin/alluxio conf get --source
+
+# Display the values in a given unit
+$ ./bin/alluxio conf get alluxio.user.block.size.bytes.default --unit KB
+$ ./bin/alluxio conf get alluxio.master.journal.flush.timeout --unit S
+`,
+		Args: cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/conf/log.go

@@ -42,7 +42,23 @@ func (c *LogCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   Log.CommandName,
 		Short: "Get or set the log level for the specified logger",
-		Args:  cobra.NoArgs,
+		Long: `The log command returns the current value of or updates the log level of a particular class on specific instances.
+Users are able to change Alluxio server-side log levels at runtime.
+
+The --target flag specifies which processes to apply the log level change to.
+The target could be of the form <master|workers|job_master|job_workers|host:webPort[:role]> and multiple targets can be listed as comma-separated entries.
+The role can be one of master,worker,job_master,job_worker.
+Using the role option is useful when an Alluxio process is configured to use a non-standard web port (e.g. if an Alluxio master does not use 19999 as its web port).
+The default target value is the primary master, primary job master, all workers and job workers.
+
+> Note: This command requires the Alluxio cluster to be running.`,
+		Example: `# Set DEBUG level for DefaultFileSystemMaster class on master processes
+$ ./bin/alluxio conf log --logName alluxio.master.file.DefaultFileSystemMaster --target=master --level=DEBUG
+
+# Set WARN level for PagedDoraWorker class on the worker process on host myHostName
+$ ./bin/alluxio conf log --logName alluxio.worker.dora.PagedDoraWorker.java --target=myHostName:worker --level=WARN
+`,
+		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/exec/test_basic_io.go

@@ -44,7 +44,12 @@ func (c *TestRunCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "basicIOTest",
 		Args:  cobra.NoArgs,
-		Short: "Run all end-to-end tests, or a specific test, on an Alluxio cluster.",
+		Short: "Run all end-to-end tests or a specific test, on an Alluxio cluster.",
+		Example: `# Run all permutations of IO tests
+$ ./bin/alluxio exec basicIOTest
+
+# Run a specific permutation of the IO tests
+$ ./bin/alluxio exec basicIOtest --operation BASIC --readType NO_CACHE --writeType THROUGH`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},
@@ -61,7 +66,7 @@ func (c *TestRunCommand) ToCommand() *cobra.Command {
 		"Alluxio worker addresses to run tests on. \n"+
 			"If not specified, random ones will be used.")
 	cmd.Flags().StringVar(&c.writeType, "writeType", "",
-		"The write type to use, one of MUST_CACHE, CACHE_THROUGH, THROUGH, ASYNC_THROUGH. \n"+
+		"The write type to use, one of MUST_CACHE, CACHE_THROUGH, THROUGH. \n"+
 			"By default all writeTypes are tested.")
 	return cmd
 }

cli/src/alluxio.org/cli/cmd/exec/test_ufs_io.go

@@ -46,10 +46,22 @@ func (c *TestUfsIOCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   c.CommandName,
 		Short: "A benchmarking tool for the I/O between Alluxio and UFS.",
-		Long: "A benchmarking tool for the I/O between Alluxio and UFS." +
-			"This test will measure the I/O throughput between Alluxio workers and the specified UFS path. " +
-			"Each worker will create concurrent clients to first generate test files of the specified size " +
-			"then read those files. The write/read I/O throughput will be measured in the process.",
+		Long: `A benchmarking tool for the I/O between Alluxio and UFS.
+This test will measure the I/O throughput between Alluxio workers and the specified UFS path.
+Each worker will create concurrent clients to first generate test files of the specified size then read those files.
+The write/read I/O throughput will be measured in the process.`,
+		Example: `# This runs the I/O benchmark to HDFS in your process locally
+$ ./bin/alluxio runUfsIOTest --path hdfs://<hdfs-address>
+
+# This invokes the I/O benchmark to HDFS in the Alluxio cluster
+# 1 worker will be used. 4 threads will be created, each writing then reading 4G of data
+$ ./bin/alluxio runUfsIOTest --path hdfs://<hdfs-address> --cluster --cluster-limit 1
+
+# This invokes the I/O benchmark to HDFS in the Alluxio cluster
+# 2 workers will be used
+# 2 threads will be created on each worker
+# Each thread is writing then reading 512m of data
+$ ./bin/alluxio runUfsIOTest --path hdfs://<hdfs-address> --cluster --cluster-limit 2 --io-size 512m --threads 2`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/fs/cat.go

@@ -39,7 +39,10 @@ func (c *CatCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "cat [path]",
 		Short: "Print specified file's content",
-		Args:  cobra.ExactArgs(1),
+		Long:  `The cat command prints the contents of a file in Alluxio to the shell.`,
+		Example: `# Print the contents of /output/part-00000
+$ ./bin/alluxio fs cat /output/part-00000`,
+		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/fs/checksum.go

@@ -39,7 +39,16 @@ func (c *ChecksumCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "checksum [path]",
 		Short: "Calculates the md5 checksum of a specified file",
-		Args:  cobra.ExactArgs(1),
+		Long: `The checksum command outputs the md5 value of a file in Alluxio.
+This can be used to verify the contents of a file stored in Alluxio.`,
+		Example: `# Compare the checksum values
+# value from Alluxio filesystem
+$ ./bin/alluxio fs checksum /LICENSE
+md5sum: bf0513403ff54711966f39b058e059a3
+# value from local filesystem
+md5 LICENSE
+MD5 (LICENSE) = bf0513403ff54711966f39b058e059a3`,
+		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/fs/chgrp.go

@@ -39,15 +39,20 @@ func (c *ChgrpCommand) Base() *env.BaseJavaCommand {
 
 func (c *ChgrpCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
-		Use:   fmt.Sprintf("%s <group> <path>", c.CommandName),
-		Short: "changes the group of a file or directory specified by args",
-		Args:  cobra.ExactArgs(2),
+		Use:   fmt.Sprintf("%s [group] [path]", c.CommandName),
+		Short: "Changes the group of a file or directory",
+		Long: `The chgrp command changes the group of the file or directory in Alluxio.
+Alluxio supports file authorization with POSIX file permissions.
+The file owner or superuser can execute this command.`,
+		Example: `# Change the group of a file
+$ ./bin/alluxio fs chgrp alluxio-group-new /input/file1`,
+		Args: cobra.ExactArgs(2),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},
 	})
 	cmd.Flags().BoolVarP(&c.recursive, "recursive", "R", false,
-		"change the group recursively")
+		"change the group recursively for all files and directories under the given path")
 	return cmd
 }
 

cli/src/alluxio.org/cli/cmd/fs/chmod.go

@@ -39,15 +39,20 @@ func (c *ChmodCommand) Base() *env.BaseJavaCommand {
 
 func (c *ChmodCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
-		Use:   fmt.Sprintf("%s <mode> <path>", c.CommandName),
-		Short: "Changes the permission of a file or directory specified by args",
-		Args:  cobra.ExactArgs(2),
+		Use:   fmt.Sprintf("%s [mode] [path]", c.CommandName),
+		Short: "Changes the permission of a file or directory",
+		Long: `The chmod command changes the permission of a file or directory in Alluxio.
+The permission mode is represented as an octal 3 digit value.
+Refer to https://en.wikipedia.org/wiki/Chmod#Numerical_permissions for a detailed description of the modes.`,
+		Example: `# Set mode 755 for /input/file
+$ ./bin/alluxio fs chmod 755 /input/file1`,
+		Args: cobra.ExactArgs(2),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},
 	})
 	cmd.Flags().BoolVarP(&c.recursive, "recursive", "R", false,
-		"change the permission recursively")
+		"change the permission recursively for all files and directories under the given path")
 	return cmd
 }
 

cli/src/alluxio.org/cli/cmd/fs/chown.go

@@ -41,13 +41,17 @@ func (c *ChownCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   fmt.Sprintf("%s <owner>[:<group>] <path>", c.CommandName),
 		Short: "Changes the owner of a file or directory specified by args",
-		Args:  cobra.NoArgs,
+		Long: `The chown command changes the owner of a file or directory in Alluxio.
+The ownership of a file can only be altered by a superuser`,
+		Example: `# Change the owner of /input/file1 to alluxio-user
+$ ./bin/alluxio fs chown alluxio-user /input/file1`,
+		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},
 	})
 	cmd.Flags().BoolVarP(&c.recursive, "recursive", "R", false,
-		"change the owner recursively")
+		"change the owner recursively for all files and directories under the given path")
 	return cmd
 }
 

cli/src/alluxio.org/cli/cmd/fs/cp.go

@@ -46,9 +46,20 @@ func (c *CopyCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "cp [srcPath] [dstPath]",
 		Short: "Copy a file or directory",
-		Long: `Copies a file or directory in the Alluxio filesystem or between local and Alluxio filesystems
-Use the file:// schema to indicate a local filesystem path (ex. file:///absolute/path/to/file) and
-use the recursive flag to copy directories`,
+		Long: `Copies a file or directory in the Alluxio filesystem or between local and Alluxio filesystems.
+The file:// scheme indicates a local filesystem path and the alluxio:// scheme or no scheme indicates an Alluxio filesystem path.`,
+		Example: `# Copy within the Alluxio filesystem
+$ ./bin/alluxio fs cp /file1 /file2
+
+# Copy a local file to the Alluxio filesystem
+$ ./bin/alluxio fs cp file:///file1 /file2
+
+# Copy a file in Alluxio to local
+$ ./bin/alluxio fs cp alluxio:///file1 file:///file2
+
+# Recursively copy a directory within the Alluxio filesystem
+$ ./bin/alluxio fs cp -R /dir1 /dir2
+`,
 		Args: cobra.ExactArgs(2),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)

cli/src/alluxio.org/cli/cmd/fs/fs.go

@@ -19,7 +19,22 @@ import (
 var Service = &env.Service{
 	Name:        "fs",
 	Description: "Operations to interface with the Alluxio filesystem",
-	Commands:    Cmds(names.FileSystemShellJavaClass),
+	Documentation: `Operations to interface with the Alluxio filesystem
+For commands that take Alluxio URIs as an argument such as ls or mkdir, the argument should be either
+- A complete Alluxio URI, such as alluxio://<masterHostname>:<masterPort>/<path>
+- A path without its scheme header, such as /path, in order to use the default hostname and port set in alluxio-site.properties
+
+> Note: All fs commands require the Alluxio cluster to be running.
+
+Most of the commands which require path components allow wildcard arguments for ease of use.
+For example, the command "bin/alluxio fs rm '/data/2014*'" deletes anything in the data directory with a prefix of 2014.
+
+Some shells will attempt to glob the input paths, causing strange errors.
+As a workaround, you can disable globbing (depending on the shell type; for example, set -f) or by escaping wildcards
+For example, the command "bin/alluxio fs cat /\\*" uses the escape backslash character twice.
+This is because the shell script will eventually call a java program which should have the final escaped parameters "cat /\\*".
+`,
+	Commands: Cmds(names.FileSystemShellJavaClass),
 }
 
 func Cmds(className string) []env.Command {

cli/src/alluxio.org/cli/cmd/fs/head.go

@@ -41,12 +41,16 @@ func (c *HeadCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "head [path]",
 		Short: "Print the leading bytes from the specified file",
-		Args:  cobra.ExactArgs(1),
+		Long: `The head command prints the first 1KB of data of a file to the shell.
+Specifying the -c flag sets the number of bytes to print.`,
+		Example: `# Print first 2048 bytes of a file
+$ ./bin/alluxio fs head -c 2048 /output/part-00000`,
+		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},
 	})
-	cmd.Flags().StringVar(&c.bytes, "bytes", "", "Byte size to print")
+	cmd.Flags().StringVarP(&c.bytes, "bytes", "c", "", "Byte size to print")
 	return cmd
 }
 

cli/src/alluxio.org/cli/cmd/fs/ls.go

@@ -70,8 +70,18 @@ func (c *LsCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "ls [path]",
 		Short: "Prints information for files and directories at the given path",
-		Long:  `Displays information for all files and directories directly under the specified paths, including permission, owner, group, size (bytes for files or the number of children for directories), persistence state, last modified time, the percentage of content already in Alluxio, and the path`,
-		Args:  cobra.ExactArgs(1),
+		Long: `The ls command lists all the immediate children in a directory and displays the file size, last modification time, and in memory status of the files.
+Using ls on a file will only display the information for that specific file.
+
+The ls command will also load the metadata for any file or immediate children of a directory from the under storage system to Alluxio namespace if it does not exist in Alluxio.
+It queries the under storage system for any file or directory matching the given path and creates a mirror of the file in Alluxio backed by that file.
+Only the metadata, such as the file name and size, are loaded this way and no data transfer occurs.`,
+		Example: `# List and load metadata for all immediate children of /s3/data
+$ ./bin/alluxio fs ls /s3/data
+
+# Force loading metadata of /s3/data
+$ ./bin/alluxio fs ls -f /s3/data`,
+		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/fs/mkdir.go

@@ -39,7 +39,15 @@ func (c *MkdirCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "mkdir [path1 path2 ...]",
 		Short: "Create directories at the specified paths, creating the parent directory if not exists",
-		Args:  cobra.MinimumNArgs(1),
+		Long: `The mkdir command creates a new directory in the Alluxio filesystem.
+It is recursive and will create any parent directories that do not exist.
+Note that the created directory will not be created in the under storage system until a file in the directory is persisted to the underlying storage.
+Using mkdir on an invalid or existing path will fail.`,
+		Example: `# Creating a folder structure
+$ ./bin/alluxio fs mkdir /users
+$ ./bin/alluxio fs mkdir /users/Alice
+$ ./bin/alluxio fs mkdir /users/Bob`,
+		Args: cobra.MinimumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},

cli/src/alluxio.org/cli/cmd/fs/mv.go

@@ -39,7 +39,13 @@ func (c *MoveCommand) ToCommand() *cobra.Command {
 	cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
 		Use:   "mv [srcPath] [dstPath]",
 		Short: "Rename a file or directory",
-		Args:  cobra.ExactArgs(2),
+		Long: `The mv command moves a file or directory to another path in Alluxio.
+The destination path must not exist or be a directory.
+If it is a directory, the file or directory will be placed as a child of the directory.
+The command is purely a metadata operation and does not affect the data blocks of the file.`,
+		Example: `# Moving a file
+$ ./bin/alluxio fs mv /data/2014 /data/archives/2014`,
+		Args: cobra.ExactArgs(2),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return c.Run(args)
 		},