diff --git a/cmd/logcli/client.go b/cmd/logcli/client.go index 3acaea6b65eea..9e31353ee69b5 100644 --- a/cmd/logcli/client.go +++ b/cmd/logcli/client.go @@ -68,7 +68,7 @@ func doRequest(path string, out interface{}) error { } defer func() { if err := resp.Body.Close(); err != nil { - fmt.Println("error closing body", err) + log.Println("error closing body", err) } }() @@ -92,7 +92,7 @@ func wsConnect(path string) (*websocket.Conn, error) { } else if strings.HasPrefix(url, "http") { url = strings.Replace(url, "http", "ws", 1) } - fmt.Println(url) + log.Println(url) h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}} c, resp, err := websocket.DefaultDialer.Dial(url, h) diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index 596bdda6f0f0e..42861a2d1b242 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -14,14 +14,17 @@ var ( username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String() password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String() - queryCmd = app.Command("query", "Run a LogQL query.") - queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String() - regexpStr = queryCmd.Arg("regex", "").String() - limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int() - since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration() - forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool() - tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool() - noLabels = queryCmd.Flag("no-labels", "Do not print labels").Default("false").Bool() + queryCmd = app.Command("query", "Run a LogQL query.") + queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String() + regexpStr = queryCmd.Arg("regex", "").String() + limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int() + since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration() + forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool() + tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool() + noLabels = queryCmd.Flag("no-labels", "Do not print any labels").Default("false").Bool() + ignoreLabelsKey = queryCmd.Flag("exclude-label", "Exclude labels given the provided key during output.").Strings() + showLabelsKey = queryCmd.Flag("include-label", "Include labels given the provided key during output.").Strings() + fixedLabelsLen = queryCmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").Int() labelsCmd = app.Command("labels", "Find values for a given label.") labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(listLabels).String() diff --git a/cmd/logcli/query.go b/cmd/logcli/query.go index b28fadb6a866a..f5b5716279477 100644 --- a/cmd/logcli/query.go +++ b/cmd/logcli/query.go @@ -1,14 +1,12 @@ package main import ( - "fmt" "log" "strings" "time" "github.com/fatih/color" "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/promql" "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" @@ -21,9 +19,8 @@ func doQuery() { } var ( - i iter.EntryIterator - common labels.Labels - maxLabelsLen = 100 + i iter.EntryIterator + common labels.Labels ) end := time.Now() @@ -43,24 +40,43 @@ func doQuery() { labelsCache := func(labels string) labels.Labels { return cache[labels] } + common = commonLabels(lss) - i = iter.NewQueryResponseIterator(resp, d) + + // Remove the labels we want to show from common + if len(*showLabelsKey) > 0 { + common = common.MatchLabels(false, *showLabelsKey...) + } if len(common) > 0 { - fmt.Println("Common labels:", color.RedString(common.String())) + log.Println("Common labels:", color.RedString(common.String())) + } + + if len(*ignoreLabelsKey) > 0 { + log.Println("Ignoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ","))) } + // Get the max size of labels + maxLabelsLen := *fixedLabelsLen for _, ls := range cache { ls = subtract(common, ls) + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } len := len(ls.String()) if maxLabelsLen < len { maxLabelsLen = len } } + i = iter.NewQueryResponseIterator(resp, d) + for i.Next() { ls := labelsCache(i.Labels()) ls = subtract(ls, common) + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } labels := "" if !*noLabels { @@ -74,95 +90,3 @@ func doQuery() { log.Fatalf("Error from iterator: %v", err) } } - -func printLogEntry(ts time.Time, lbls string, line string) { - fmt.Println( - color.BlueString(ts.Format(time.RFC3339)), - color.RedString(lbls), - strings.TrimSpace(line), - ) -} - -func padLabel(ls labels.Labels, maxLabelsLen int) string { - labels := ls.String() - if len(labels) < maxLabelsLen { - labels += strings.Repeat(" ", maxLabelsLen-len(labels)) - } - return labels -} - -func mustParseLabels(labels string) labels.Labels { - ls, err := promql.ParseMetric(labels) - if err != nil { - log.Fatalf("Failed to parse labels: %+v", err) - } - return ls -} - -func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) { - cache := make(map[string]labels.Labels, len(resp.Streams)) - lss := make([]labels.Labels, 0, len(resp.Streams)) - for _, stream := range resp.Streams { - ls := mustParseLabels(stream.Labels) - cache[stream.Labels] = ls - lss = append(lss, ls) - } - return cache, lss -} - -func commonLabels(lss []labels.Labels) labels.Labels { - if len(lss) == 0 { - return nil - } - - result := lss[0] - for i := 1; i < len(lss); i++ { - result = intersect(result, lss[i]) - } - return result -} - -func intersect(a, b labels.Labels) labels.Labels { - var result labels.Labels - for i, j := 0, 0; i < len(a) && j < len(b); { - k := strings.Compare(a[i].Name, b[j].Name) - switch { - case k == 0: - if a[i].Value == b[j].Value { - result = append(result, a[i]) - } - i++ - j++ - case k < 0: - i++ - case k > 0: - j++ - } - } - return result -} - -// subtract b from a -func subtract(a, b labels.Labels) labels.Labels { - var result labels.Labels - i, j := 0, 0 - for i < len(a) && j < len(b) { - k := strings.Compare(a[i].Name, b[j].Name) - if k != 0 || a[i].Value != b[j].Value { - result = append(result, a[i]) - } - switch { - case k == 0: - i++ - j++ - case k < 0: - i++ - case k > 0: - j++ - } - } - for ; i < len(a); i++ { - result = append(result, a[i]) - } - return result -} diff --git a/cmd/logcli/query_test.go b/cmd/logcli/query_test.go new file mode 100644 index 0000000000000..a8a3db231df97 --- /dev/null +++ b/cmd/logcli/query_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "reflect" + "testing" + + "github.com/prometheus/prometheus/pkg/labels" +) + +func Test_commonLabels(t *testing.T) { + type args struct { + lss []labels.Labels + } + tests := []struct { + name string + args args + want labels.Labels + }{ + { + "Extract common labels source > target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`)}, + }, + mustParseLabels(`{bar="foo"}`), + }, + { + "Extract common labels source > target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="bar", baz="baz"}`)}, + }, + mustParseLabels(`{foo="bar", bar="foo"}`), + }, + { + "Extract common labels source < target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo"}`)}, + }, + mustParseLabels(`{bar="foo"}`), + }, + { + "Extract common labels source < target no common", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{fo="bar"}`)}, + }, + labels.Labels{}, + }, + { + "Extract common labels source = target no common", + args{ + []labels.Labels{mustParseLabels(`{foo="bar"}`), mustParseLabels(`{fooo="bar"}`)}, + }, + labels.Labels{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := commonLabels(tt.args.lss); !reflect.DeepEqual(got, tt.want) { + t.Errorf("commonLabels() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_subtract(t *testing.T) { + type args struct { + a labels.Labels + b labels.Labels + } + tests := []struct { + name string + args args + want labels.Labels + }{ + { + "Subtract labels source > target", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source < target", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{bar="foo"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source < target no sub", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{fo="bar"}`), + }, + mustParseLabels(`{bar="foo", foo="bar"}`), + }, + { + "Subtract labels source = target no sub", + args{ + mustParseLabels(`{foo="bar"}`), + mustParseLabels(`{fiz="buz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source > target no sub", + args{ + mustParseLabels(`{foo="bar"}`), + mustParseLabels(`{fiz="buz", foo="baz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source > target no sub", + args{ + mustParseLabels(`{a="b", foo="bar", baz="baz", fizz="fizz"}`), + mustParseLabels(`{foo="bar", baz="baz", buzz="buzz", fizz="fizz"}`), + }, + mustParseLabels(`{a="b"}`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := subtract(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { + t.Errorf("subtract() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/logcli/tail.go b/cmd/logcli/tail.go index 51e64bd221e0e..27331aa58a104 100644 --- a/cmd/logcli/tail.go +++ b/cmd/logcli/tail.go @@ -2,6 +2,9 @@ package main import ( "log" + "strings" + + "github.com/fatih/color" "github.com/grafana/loki/pkg/logproto" ) @@ -14,6 +17,14 @@ func tailQuery() { stream := new(logproto.Stream) + if len(*ignoreLabelsKey) > 0 { + log.Println("Ingoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ","))) + } + + if len(*showLabelsKey) > 0 { + log.Println("Print only labels key:", color.RedString(strings.Join(*showLabelsKey, ","))) + } + for { err := conn.ReadJSON(stream) if err != nil { @@ -23,7 +34,25 @@ func tailQuery() { labels := "" if !*noLabels { - labels = stream.Labels + + if len(*ignoreLabelsKey) > 0 || len(*showLabelsKey) > 0 { + + ls := mustParseLabels(stream.GetLabels()) + + if len(*showLabelsKey) > 0 { + ls = ls.MatchLabels(true, *showLabelsKey...) + } + + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } + + labels = ls.String() + + } else { + + labels = stream.Labels + } } for _, entry := range stream.Entries { printLogEntry(entry.Timestamp, labels, entry.Line) diff --git a/cmd/logcli/utils.go b/cmd/logcli/utils.go new file mode 100644 index 0000000000000..807c3a6184873 --- /dev/null +++ b/cmd/logcli/utils.go @@ -0,0 +1,109 @@ +package main + +import ( + "fmt" + "log" + "sort" + "strings" + "time" + + "github.com/fatih/color" + "github.com/grafana/loki/pkg/logproto" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/promql" +) + +// print a log entry +func printLogEntry(ts time.Time, lbls string, line string) { + fmt.Println( + color.BlueString(ts.Format(time.RFC3339)), + color.RedString(lbls), + strings.TrimSpace(line), + ) +} + +// add some padding after labels +func padLabel(ls labels.Labels, maxLabelsLen int) string { + labels := ls.String() + if len(labels) < maxLabelsLen { + labels += strings.Repeat(" ", maxLabelsLen-len(labels)) + } + return labels +} + +// parse labels from string +func mustParseLabels(labels string) labels.Labels { + ls, err := promql.ParseMetric(labels) + if err != nil { + log.Fatalf("Failed to parse labels: %+v", err) + } + return ls +} + +// parse labels from response stream +func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) { + cache := make(map[string]labels.Labels, len(resp.Streams)) + lss := make([]labels.Labels, 0, len(resp.Streams)) + for _, stream := range resp.Streams { + ls := mustParseLabels(stream.Labels) + cache[stream.Labels] = ls + lss = append(lss, ls) + } + return cache, lss +} + +// return common labels between given lavels set +func commonLabels(lss []labels.Labels) labels.Labels { + if len(lss) == 0 { + return nil + } + + result := lss[0] + for i := 1; i < len(lss); i++ { + result = intersect(result, lss[i]) + } + return result +} + +// intersect two labels set +func intersect(a, b labels.Labels) labels.Labels { + + set := labels.Labels{} + ma := a.Map() + mb := b.Map() + + for ka, va := range ma { + if vb, ok := mb[ka]; ok { + if vb == va { + set = append(set, labels.Label{ + Name: ka, + Value: va, + }) + } + } + } + sort.Sort(set) + return set +} + +// subtract labels set b from labels set a +func subtract(a, b labels.Labels) labels.Labels { + + set := labels.Labels{} + ma := a.Map() + mb := b.Map() + + for ka, va := range ma { + if vb, ok := mb[ka]; ok { + if vb == va { + continue + } + } + set = append(set, labels.Label{ + Name: ka, + Value: va, + }) + } + sort.Sort(set) + return set +} diff --git a/docs/logcli.md b/docs/logcli.md index d7330a2826bbb..b72727867bb6c 100644 --- a/docs/logcli.md +++ b/docs/logcli.md @@ -81,15 +81,20 @@ usage: logcli query [] [] Run a LogQL query. Flags: - --help Show context-sensitive help (also try --help-long and --help-man). - --addr="" Server address, need to specify. - --username="" Username for HTTP basic auth. - --password="" Password for HTTP basic auth. - --limit=30 Limit on number of entries to print. - --since=1h Lookback window. - --forward Scan forwards through logs. - -t, --tail Tail the logs - --no-labels Do not print labels + --help Show context-sensitive help (also try --help-long and --help-man). + --addr="" Server address, need to specify. + --username="" Username for HTTP basic auth. + --password="" Password for HTTP basic auth. + --limit=30 Limit on number of entries to print. + --since=1h Lookback window. + --forward Scan forwards through logs. + -t, --tail Tail the logs + --no-labels Do not print any labels + --exclude-label=EXCLUDE-LABEL ... + Exclude labels given the provided key during output. + --include-label=INCLUDE-LABEL ... + Include labels given the provided key during output. + --labels-length=0 Set a fixed padding to labels Args: eg '{foo="bar",baz="blip"}'