Add flags to recursively list secrets for KV Engine.

command/base_helpers.go

@@ -50,7 +50,7 @@ func ensureTrailingSlash(s string) string {
 	return s
 }
 
-// ensureNoTrailingSlash ensures the given string has a trailing slash.
+// ensureNoTrailingSlash ensures the given string has no trailing slash.
 func ensureNoTrailingSlash(s string) string {
 	s = strings.TrimSpace(s)
 	if s == "" {
@@ -63,7 +63,7 @@ func ensureNoTrailingSlash(s string) string {
 	return s
 }
 
-// ensureNoLeadingSlash ensures the given string has a trailing slash.
+// ensureNoLeadingSlash ensures the given string has no leading slash.
 func ensureNoLeadingSlash(s string) string {
 	s = strings.TrimSpace(s)
 	if s == "" {

command/kv_helpers.go

@@ -5,12 +5,40 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"regexp"
 	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
 
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/sdk/helper/strutil"
 )
 
+// kvData is a helper struct for `listRecursive'.
+type kvData struct {
+	path   string      // Name (path) of the secret.
+	secret *api.Secret // Data belonging to the secret (sub-paths too).
+	err    error       // Errors when fetching the secret (if any).
+}
+
+// kvRecursiveListParams contains a list of parameters
+// to be passed to `kvListRecursive'.
+type kvListRecursiveParams struct {
+	v2     *bool       // Flag to store if the API is v2.
+	client *api.Client // Client object to use for fetching secrets.
+
+	data   []*kvData      // List of all the secrets fetched recursively.
+	depth  int32          // Depth of the recursion.
+	track  int32          // Helper variable to track the recursion depth.
+	filter *regexp.Regexp // Filter to be applied for matching keys.
+
+	wg  sync.WaitGroup // Keep track of launched goroutines.
+	sem chan int32     // Semaphone for throttling goroutines.
+	mux sync.Mutex     // Mutex to append entries to `data'.
+	tck *time.Ticker   // Avoid busy loops when polling for goroutines.
+}
+
 func kvReadRequest(client *api.Client, path string, params map[string]string) (*api.Secret, error) {
 	r := client.NewRequest("GET", "/v1/"+path)
 	for k, v := range params {
@@ -115,6 +143,10 @@ func addPrefixToVKVPath(p, mountPath, apiPrefix string) string {
 	}
 }
 
+func removePrefixFromVKVPath(p, apiPrefix string) string {
+	return strings.Replace(p, apiPrefix, "", 1)
+}
+
 func getHeaderForMap(header string, data map[string]interface{}) string {
 	maxKey := 0
 	for k := range data {
@@ -150,3 +182,68 @@ func kvParseVersionsFlags(versions []string) []string {
 
 	return versionsOut
 }
+
+// kvListRecursive is a helper function for listing paths
+// upto a given depth, recursively.
+func kvListRecursive(r *kvListRecursiveParams, base string, sub []interface{}) {
+	defer func() {
+		// Increment the go-routine tracker.
+		atomic.AddInt32(&r.track, 1)
+
+		// Decrement the wait-group count.
+		r.wg.Done()
+	}()
+
+	// Wait in the queue.
+	r.sem <- int32(1)
+
+	// No need to recursively list values that don't have any sub-paths.
+	if !strings.HasSuffix(base, "/") {
+		return
+	}
+
+	// Strip trailing slash, because we want to append
+	// it with each of the children's paths.
+	base = ensureNoTrailingSlash(base)
+
+	// Calculate the recursion depth of this call.
+	d := int32(strings.Count(base, "/"))
+	if *r.v2 {
+		d--
+	}
+
+	// Base case for return.
+	if len(sub) <= 0 || (r.depth != -1 && d >= r.depth) {
+		return
+	}
+
+	var tmp string
+
+	// For each child, construct the new path and call recursively.
+	for _, p := range sub {
+		path := fmt.Sprintf("%s/%s", base, p)
+		secret, err := r.client.Logical().List(path)
+
+		tmp = path
+		if *r.v2 {
+			tmp = removePrefixFromVKVPath(tmp, "metadata/")
+		}
+
+		// Append the entry (that matches the filter) to the list.
+		if r.filter.MatchString(tmp) {
+			r.mux.Lock()
+			r.data = append(r.data, &kvData{tmp, secret, err})
+			r.mux.Unlock()
+		}
+
+		if err != nil {
+			continue
+		}
+
+		// Only call if there are children.
+		if s, ok := extractListData(secret); ok {
+			r.wg.Add(1)
+			go kvListRecursive(r, path, s)
+		}
+	}
+}

command/kv_list.go

@@ -2,7 +2,12 @@ package command
 
 import (
 	"fmt"
+	"regexp"
+	"sort"
 	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
 
 	"github.com/mitchellh/cli"
 	"github.com/posener/complete"
@@ -13,6 +18,11 @@ var _ cli.CommandAutocomplete = (*KVListCommand)(nil)
 
 type KVListCommand struct {
 	*BaseCommand
+
+	flagDepth      int
+	flagFilter     string
+	flagRecursive  bool
+	flagConcurrent uint
 }
 
 func (c *KVListCommand) Synopsis() string {
@@ -38,7 +48,39 @@ Usage: vault kv list [options] PATH
 }
 
 func (c *KVListCommand) Flags() *FlagSets {
-	return c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+
+	f := set.NewFlagSet("Command Options")
+
+	f.BoolVar(&BoolVar{
+		Name:    "recursive",
+		Target:  &c.flagRecursive,
+		Default: false,
+		Usage:   "Recursively list data for a given path.",
+	})
+
+	f.IntVar(&IntVar{
+		Name:    "depth",
+		Target:  &c.flagDepth,
+		Default: -1,
+		Usage:   "Specifies the depth for recursive listing.",
+	})
+
+	f.StringVar(&StringVar{
+		Name:    "filter",
+		Target:  &c.flagFilter,
+		Default: `.*`,
+		Usage:   "Specifies a regular expression for filtering paths.",
+	})
+
+	f.UintVar(&UintVar{
+		Name:    "concurrent",
+		Target:  &c.flagConcurrent,
+		Default: 16,
+		Usage:   "Specifies the number of concurrent recursions to run.",
+	})
+
+	return set
 }
 
 func (c *KVListCommand) AutocompleteArgs() complete.Predictor {
@@ -67,6 +109,25 @@ func (c *KVListCommand) Run(args []string) int {
 		return 1
 	}
 
+	if c.flagRecursive && c.flagDepth < -1 {
+		c.UI.Error(fmt.Sprintf("Invalid recursion depth: %d", c.flagDepth))
+		return 1
+	}
+
+	if _, e := regexp.Compile(c.flagFilter); c.flagRecursive && e != nil {
+		c.UI.Error(fmt.Sprintf(
+			"Invalid regular expression: %s", c.flagFilter,
+		))
+		return 1
+	}
+
+	if c.flagRecursive && c.flagConcurrent <= 0 {
+		c.UI.Error(fmt.Sprintf(
+			"Invalid concurrency value: %d", c.flagConcurrent,
+		))
+		return 1
+	}
+
 	client, err := c.Client()
 	if err != nil {
 		c.UI.Error(err.Error())
@@ -103,10 +164,98 @@ func (c *KVListCommand) Run(args []string) int {
 		return OutputSecret(c.UI, secret)
 	}
 
-	if _, ok := extractListData(secret); !ok {
+	s, ok := extractListData(secret)
+	if !ok {
 		c.UI.Error(fmt.Sprintf("No entries found at %s", path))
 		return 2
 	}
 
+	// If we have to list keys recursively.
+	if c.flagRecursive {
+		if c.flagDepth != 0 {
+			var (
+				i = int32(1)
+				r = &kvListRecursiveParams{
+					v2:     &v2,
+					client: client,
+					data:   []*kvData{},
+					depth:  int32(c.flagDepth),
+					track:  int32(0),
+					filter: regexp.MustCompile(c.flagFilter),
+					wg:     sync.WaitGroup{},
+					sem:    make(chan int32, c.flagConcurrent),
+					mux:    sync.Mutex{},
+					tck:    time.NewTicker(time.Millisecond * 100),
+				}
+				e      bool
+				opErr  string
+				opList = []string{}
+			)
+
+			// One of the base cases for `kvListRecursive()'.
+			path = ensureTrailingSlash(path)
+
+			// Append the first entry (only for tabular format).
+			if Format(c.UI) == "table" {
+				if v2 {
+					r.data = append(
+						r.data,
+						&kvData{
+							removePrefixFromVKVPath(path, "metadata/"),
+							secret,
+							nil,
+						},
+					)
+				} else {
+					r.data = append(r.data, &kvData{path, secret, nil})
+				}
+			}
+
+			// Launch the recursive call and wait for it them to finish.
+			r.wg.Add(1)
+			go kvListRecursive(r, path, s)
+			for len(r.sem) > 0 || atomic.LoadInt32(&r.track) < i {
+				// For loop termination.
+				if atomic.LoadInt32(&r.track) == 0 {
+					atomic.AddInt32(&r.track, 1)
+				}
+
+				select {
+				case x, ok := <-r.sem:
+					if ok {
+						// For continuing the loop.
+						i += x
+					} else {
+						break
+					}
+				default:
+					<-r.tck.C
+				}
+			}
+			r.wg.Wait()
+
+			// Print the entries.
+			for _, d := range r.data {
+				if d.path != "" {
+					opList = append(opList, d.path)
+					if d.err != nil {
+						e = true
+						opErr += fmt.Sprintf("\n\t%s: %s\n", d.path, d.err)
+					}
+				}
+			}
+
+			sort.Strings(opList)
+			OutputList(c.UI, opList)
+
+			if e {
+				c.UI.Error(fmt.Sprintf("Errors:%s", opErr))
+				return 3
+			}
+			return 0
+		}
+		return 0
+	}
+
 	return OutputList(c.UI, secret)
 }