Implemented the tag values iterator for SHOW TAG VALUES

`SHOW TAG VALUES` output has been modified to print the measurement name
for every measurement and to return the output in two columns: key and
value. An example output might be:

    > SHOW TAG VALUES WITH KEY IN (host, region)
    name: cpu
    ---------
    key     value
    host    server01
    region  useast

    name: mem
    ---------
    key     value
    host    server02
    region  useast

`measurementsByExpr` has been taught how to handle reserved keys (ones
with an underscore at the beginning) to allow reusing that function and
skipping over expressions that don't matter to the call.

Fixes #5593.

CHANGELOG.md

@@ -19,6 +19,7 @@
 - [#5691](https://github.com/influxdata/influxdb/pull/5691): Remove associated shard data when retention policies are dropped.
 - [#5758](https://github.com/influxdata/influxdb/pull/5758): TSM engine stats for cache, WAL, and filestore. Thanks @jonseymour
 - [#5844](https://github.com/influxdata/influxdb/pull/5844): Tag TSM engine stats with database and retention policy
+- [#5593](https://github.com/influxdata/influxdb/issues/5593): Modify `SHOW TAG VALUES` output for the new query engine to normalize the output.
 
 ### Bugfixes
 

cmd/influxd/run/server_test.go

@@ -4973,37 +4973,37 @@ func TestServer_Query_ShowTagKeys(t *testing.T) {
 		&Query{
 			name:    "show tag values with key",
 			command: "SHOW TAG VALUES WITH KEY = host",
-			exp:     `{"results":[{"series":[{"name":"hostTagValues","columns":["host"],"values":[["server01"],["server02"],["server03"]]}]}]}`,
+			exp:     `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"],["host","server02"]]},{"name":"disk","columns":["key","value"],"values":[["host","server03"]]},{"name":"gpu","columns":["key","value"],"values":[["host","server02"],["host","server03"]]}]}]}`,
 			params:  url.Values{"db": []string{"db0"}},
 		},
 		&Query{
 			name:    `show tag values with key and where`,
 			command: `SHOW TAG VALUES FROM cpu WITH KEY = host WHERE region = 'uswest'`,
-			exp:     `{"results":[{"series":[{"name":"hostTagValues","columns":["host"],"values":[["server01"]]}]}]}`,
+			exp:     `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"]]}]}]}`,
 			params:  url.Values{"db": []string{"db0"}},
 		},
 		&Query{
-			name:    `show tag values with key and where matches regular expression`,
+			name:    `show tag values with key and where matches the regular expression`,
 			command: `SHOW TAG VALUES WITH KEY = host WHERE region =~ /ca.*/`,
-			exp:     `{"results":[{"series":[{"name":"hostTagValues","columns":["host"],"values":[["server03"]]}]}]}`,
+			exp:     `{"results":[{"series":[{"name":"disk","columns":["key","value"],"values":[["host","server03"]]},{"name":"gpu","columns":["key","value"],"values":[["host","server03"]]}]}]}`,
 			params:  url.Values{"db": []string{"db0"}},
 		},
 		&Query{
-			name:    `show tag values with key and where does not matche regular expression`,
+			name:    `show tag values with key and where does not match the regular expression`,
 			command: `SHOW TAG VALUES WITH KEY = region WHERE host !~ /server0[12]/`,
-			exp:     `{"results":[{"series":[{"name":"regionTagValues","columns":["region"],"values":[["caeast"]]}]}]}`,
+			exp:     `{"results":[{"series":[{"name":"disk","columns":["key","value"],"values":[["region","caeast"]]}]}]}`,
 			params:  url.Values{"db": []string{"db0"}},
 		},
 		&Query{
-			name:    `show tag values with key in and where does not matche regular expression`,
+			name:    `show tag values with key in and where does not match the regular expression`,
 			command: `SHOW TAG VALUES FROM cpu WITH KEY IN (host, region) WHERE region = 'uswest'`,
-			exp:     `{"results":[{"series":[{"name":"hostTagValues","columns":["host"],"values":[["server01"]]},{"name":"regionTagValues","columns":["region"],"values":[["uswest"]]}]}]}`,
+			exp:     `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"],["region","uswest"]]}]}]}`,
 			params:  url.Values{"db": []string{"db0"}},
 		},
 		&Query{
 			name:    `show tag values with key and measurement matches regular expression`,
 			command: `SHOW TAG VALUES FROM /[cg]pu/ WITH KEY = host`,
-			exp:     `{"results":[{"series":[{"name":"hostTagValues","columns":["host"],"values":[["server01"],["server02"],["server03"]]}]}]}`,
+			exp:     `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"],["host","server02"]]},{"name":"gpu","columns":["key","value"],"values":[["host","server02"],["host","server03"]]}]}]}`,
 			params:  url.Values{"db": []string{"db0"}},
 		},
 		&Query{

influxql/ast.go

@@ -3397,7 +3397,11 @@ func Rewrite(r Rewriter, node Node) Node {
 		n.Fields = Rewrite(r, n.Fields).(Fields)
 		n.Dimensions = Rewrite(r, n.Dimensions).(Dimensions)
 		n.Sources = Rewrite(r, n.Sources).(Sources)
-		n.Condition = Rewrite(r, n.Condition).(Expr)
+		if cond := Rewrite(r, n.Condition); cond != nil {
+			n.Condition = cond.(Expr)
+		} else {
+			n.Condition = nil
+		}
 
 	case Fields:
 		for i, f := range n {
@@ -3416,11 +3420,25 @@ func Rewrite(r Rewriter, node Node) Node {
 		n.Expr = Rewrite(r, n.Expr).(Expr)
 
 	case *BinaryExpr:
-		n.LHS = Rewrite(r, n.LHS).(Expr)
-		n.RHS = Rewrite(r, n.RHS).(Expr)
+		lhs := Rewrite(r, n.LHS)
+		rhs := Rewrite(r, n.RHS)
+		if lhs != nil && rhs != nil {
+			n.LHS = lhs.(Expr)
+			n.RHS = rhs.(Expr)
+		} else if lhs != nil {
+			return lhs
+		} else if rhs != nil {
+			return rhs
+		} else {
+			return nil
+		}
 
 	case *ParenExpr:
-		n.Expr = Rewrite(r, n.Expr).(Expr)
+		if expr := Rewrite(r, n.Expr); expr != nil {
+			n.Expr = expr.(Expr)
+		} else {
+			return nil
+		}
 
 	case *Call:
 		for i, expr := range n.Args {

influxql/statement_rewriter.go

@@ -13,6 +13,8 @@ func RewriteStatement(stmt Statement) (Statement, error) {
 		return rewriteShowMeasurementsStatement(stmt)
 	case *ShowTagKeysStatement:
 		return rewriteShowTagKeysStatement(stmt)
+	case *ShowTagValuesStatement:
+		return rewriteShowTagValuesStatement(stmt)
 	default:
 		return stmt, nil
 	}
@@ -149,3 +151,88 @@ func rewriteShowTagKeysStatement(stmt *ShowTagKeysStatement) (Statement, error)
 		Dedupe:     true,
 	}, nil
 }
+
+func rewriteShowTagValuesStatement(stmt *ShowTagValuesStatement) (Statement, error) {
+	// Check for time in WHERE clause (not supported).
+	if HasTimeExpr(stmt.Condition) {
+		return nil, errors.New("SHOW TAG VALUES doesn't support time in WHERE clause")
+	}
+
+	condition := stmt.Condition
+	if len(stmt.Sources) > 0 {
+		if source, ok := stmt.Sources[0].(*Measurement); ok {
+			var expr Expr
+			if source.Regex != nil {
+				expr = &BinaryExpr{
+					Op:  EQREGEX,
+					LHS: &VarRef{Val: "name"},
+					RHS: &RegexLiteral{Val: source.Regex.Val},
+				}
+			} else if source.Name != "" {
+				expr = &BinaryExpr{
+					Op:  EQ,
+					LHS: &VarRef{Val: "name"},
+					RHS: &StringLiteral{Val: source.Name},
+				}
+			}
+
+			// Set condition or "AND" together.
+			if condition == nil {
+				condition = expr
+			} else {
+				condition = &BinaryExpr{Op: AND, LHS: expr, RHS: condition}
+			}
+		}
+	}
+
+	if len(stmt.TagKeys) > 0 {
+		var expr Expr
+		for _, tagKey := range stmt.TagKeys {
+			if expr != nil {
+				expr = &BinaryExpr{
+					Op:  OR,
+					LHS: expr,
+					RHS: &BinaryExpr{
+						Op:  EQ,
+						LHS: &VarRef{Val: "_tagKey"},
+						RHS: &StringLiteral{Val: tagKey},
+					},
+				}
+			} else {
+				expr = &BinaryExpr{
+					Op:  EQ,
+					LHS: &VarRef{Val: "_tagKey"},
+					RHS: &StringLiteral{Val: tagKey},
+				}
+			}
+		}
+
+		// Wrap in parenthesis if we have more than 1 tag key.
+		if len(stmt.TagKeys) > 1 {
+			expr = &ParenExpr{Expr: expr}
+		}
+
+		// Set condition or "AND" together.
+		if condition == nil {
+			condition = expr
+		} else {
+			condition = &BinaryExpr{Op: AND, LHS: condition, RHS: expr}
+		}
+	}
+
+	return &SelectStatement{
+		Fields: []*Field{
+			{Expr: &VarRef{Val: "_tagKey"}, Alias: "key"},
+			{Expr: &VarRef{Val: "value"}},
+		},
+		Sources: []Source{
+			&Measurement{Name: "_tags"},
+		},
+		Condition:  condition,
+		Offset:     stmt.Offset,
+		Limit:      stmt.Limit,
+		SortFields: stmt.SortFields,
+		OmitTime:   true,
+		Dedupe:     true,
+	}, nil
+}

influxql/statement_rewriter_test.go

@@ -59,6 +59,18 @@ func TestRewriteStatement(t *testing.T) {
 			stmt: `SHOW TAG KEYS FROM cpu WHERE region = 'uswest'`,
 			s:    `SELECT tagKey FROM _tagKeys WHERE "name" = 'cpu' AND region = 'uswest'`,
 		},
+		{
+			stmt: `SHOW TAG VALUES WITH KEY = region`,
+			s:    `SELECT _tagKey AS "key", value FROM _tags WHERE _tagKey = 'region'`,
+		},
+		{
+			stmt: `SHOW TAG VALUES FROM cpu WITH KEY = region`,
+			s:    `SELECT _tagKey AS "key", value FROM _tags WHERE "name" = 'cpu' AND _tagKey = 'region'`,
+		},
+		{
+			stmt: `SHOW TAG VALUES FROM cpu WITH KEY IN (region, host)`,
+			s:    `SELECT _tagKey AS "key", value FROM _tags WHERE "name" = 'cpu' AND (_tagKey = 'region' OR _tagKey = 'host')`,
+		},
 		{
 			stmt: `SELECT value FROM cpu`,
 			s:    `SELECT value FROM cpu`,

tsdb/meta.go

@@ -142,16 +142,16 @@ func (d *DatabaseIndex) TagsForSeries(key string) map[string]string {
 	return ss.Tags
 }
 
-// measurementsByExpr takes and expression containing only tags and returns
+// measurementsByExpr takes an expression containing only tags and returns
 // a list of matching *Measurement.
-func (d *DatabaseIndex) measurementsByExpr(expr influxql.Expr) (Measurements, error) {
+func (d *DatabaseIndex) measurementsByExpr(expr influxql.Expr) (Measurements, bool, error) {
 	switch e := expr.(type) {
 	case *influxql.BinaryExpr:
 		switch e.Op {
 		case influxql.EQ, influxql.NEQ, influxql.EQREGEX, influxql.NEQREGEX:
 			tag, ok := e.LHS.(*influxql.VarRef)
 			if !ok {
-				return nil, fmt.Errorf("left side of '%s' must be a tag key", e.Op.String())
+				return nil, false, fmt.Errorf("left side of '%s' must be a tag key", e.Op.String())
 			}
 
 			tf := &TagFilter{
@@ -162,46 +162,55 @@ func (d *DatabaseIndex) measurementsByExpr(expr influxql.Expr) (Measurements, er
 			if influxql.IsRegexOp(e.Op) {
 				re, ok := e.RHS.(*influxql.RegexLiteral)
 				if !ok {
-					return nil, fmt.Errorf("right side of '%s' must be a regular expression", e.Op.String())
+					return nil, false, fmt.Errorf("right side of '%s' must be a regular expression", e.Op.String())
 				}
 				tf.Regex = re.Val
 			} else {
 				s, ok := e.RHS.(*influxql.StringLiteral)
 				if !ok {
-					return nil, fmt.Errorf("right side of '%s' must be a tag value string", e.Op.String())
+					return nil, false, fmt.Errorf("right side of '%s' must be a tag value string", e.Op.String())
 				}
 				tf.Value = s.Val
 			}
 
 			// Match on name, if specified.
 			if tag.Val == "name" {
-				return d.measurementsByNameFilter(tf.Op, tf.Value, tf.Regex), nil
+				return d.measurementsByNameFilter(tf.Op, tf.Value, tf.Regex), true, nil
+			} else if strings.HasPrefix(tag.Val, "_") {
+				return nil, false, nil
 			}
 
-			return d.measurementsByTagFilters([]*TagFilter{tf}), nil
+			return d.measurementsByTagFilters([]*TagFilter{tf}), true, nil
 		case influxql.OR, influxql.AND:
-			lhsIDs, err := d.measurementsByExpr(e.LHS)
+			lhsIDs, lhsOk, err := d.measurementsByExpr(e.LHS)
 			if err != nil {
-				return nil, err
+				return nil, false, err
 			}
 
-			rhsIDs, err := d.measurementsByExpr(e.RHS)
+			rhsIDs, rhsOk, err := d.measurementsByExpr(e.RHS)
 			if err != nil {
-				return nil, err
+				return nil, false, err
 			}
 
-			if e.Op == influxql.OR {
-				return lhsIDs.union(rhsIDs), nil
-			}
+			if lhsOk && rhsOk {
+				if e.Op == influxql.OR {
+					return lhsIDs.union(rhsIDs), true, nil
+				}
 
-			return lhsIDs.intersect(rhsIDs), nil
+				return lhsIDs.intersect(rhsIDs), true, nil
+			} else if lhsOk {
+				return lhsIDs, true, nil
+			} else if rhsOk {
+				return rhsIDs, true, nil
+			}
+			return nil, false, nil
 		default:
-			return nil, fmt.Errorf("invalid operator")
+			return nil, false, fmt.Errorf("invalid operator")
 		}
 	case *influxql.ParenExpr:
 		return d.measurementsByExpr(e.Expr)
 	}
-	return nil, fmt.Errorf("%#v", expr)
+	return nil, false, fmt.Errorf("%#v", expr)
 }
 
 // measurementsByNameFilter returns the sorted measurements matching a name.
@@ -929,6 +938,95 @@ func (m *Measurement) seriesIDsAllOrByExpr(expr influxql.Expr) (SeriesIDs, error
 	return ids, nil
 }
 
+// tagKeysByExpr extracts the tag keys wanted by the expression.
+func (m *Measurement) tagKeysByExpr(expr influxql.Expr) (stringSet, bool, error) {
+	switch e := expr.(type) {
+	case *influxql.BinaryExpr:
+		switch e.Op {
+		case influxql.EQ, influxql.NEQ, influxql.EQREGEX, influxql.NEQREGEX:
+			tag, ok := e.LHS.(*influxql.VarRef)
+			if !ok {
+				return nil, false, fmt.Errorf("left side of '%s' must be a tag key", e.Op.String())
+			}
+
+			if tag.Val != "_tagKey" {
+				return nil, false, nil
+			}
+
+			tf := TagFilter{
+				Op: e.Op,
+			}
+
+			if influxql.IsRegexOp(e.Op) {
+				re, ok := e.RHS.(*influxql.RegexLiteral)
+				if !ok {
+					return nil, false, fmt.Errorf("right side of '%s' must be a regular expression", e.Op.String())
+				}
+				tf.Regex = re.Val
+			} else {
+				s, ok := e.RHS.(*influxql.StringLiteral)
+				if !ok {
+					return nil, false, fmt.Errorf("right side of '%s' must be a tag value string", e.Op.String())
+				}
+				tf.Value = s.Val
+			}
+			return m.tagKeysByFilter(tf.Op, tf.Value, tf.Regex), true, nil
+		case influxql.AND, influxql.OR:
+			lhsKeys, lhsOk, err := m.tagKeysByExpr(e.LHS)
+			if err != nil {
+				return nil, false, err
+			}
+
+			rhsKeys, rhsOk, err := m.tagKeysByExpr(e.RHS)
+			if err != nil {
+				return nil, false, err
+			}
+
+			if lhsOk && rhsOk {
+				if e.Op == influxql.OR {
+					return lhsKeys.union(rhsKeys), true, nil
+				}
+
+				return lhsKeys.intersect(rhsKeys), true, nil
+			} else if lhsOk {
+				return lhsKeys, true, nil
+			} else if rhsOk {
+				return rhsKeys, true, nil
+			}
+			return nil, false, nil
+		default:
+			return nil, false, fmt.Errorf("invalid operator")
+		}
+	case *influxql.ParenExpr:
+		return m.tagKeysByExpr(e.Expr)
+	}
+	return nil, false, fmt.Errorf("%#v", expr)
+}
+
+// tagKeysByFilter will filter the tag keys for the measurement.
+func (m *Measurement) tagKeysByFilter(op influxql.Token, val string, regex *regexp.Regexp) stringSet {
+	ss := newStringSet()
+	for _, key := range m.TagKeys() {
+		var matched bool
+		switch op {
+		case influxql.EQ:
+			matched = key == val
+		case influxql.NEQ:
+			matched = key != val
+		case influxql.EQREGEX:
+			matched = regex.MatchString(key)
+		case influxql.NEQREGEX:
+			matched = !regex.MatchString(key)
+		}
+
+		if !matched {
+			continue
+		}
+		ss.add(key)
+	}
+	return ss
+}
+
 // tagValuer is used during expression expansion to evaluate all sets of tag values.
 type tagValuer struct {
 	tags map[string]*string