Loki HTTP/JSON Model Layer (#1022)

* First pass data model Signed-off-by: Joe Elliott <number101010@gmail.com> * Use prom model b/c we're serializing promql objects Signed-off-by: Joe Elliott <number101010@gmail.com> * Added legacy query support and tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Added legacy label test Signed-off-by: Joe Elliott <number101010@gmail.com> * Added tail response marshalling and tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed marshallers and test Signed-off-by: Joe Elliott <number101010@gmail.com> * Expanded legacy test cases Signed-off-by: Joe Elliott <number101010@gmail.com> * Dropped streams nano test Signed-off-by: Joe Elliott <number101010@gmail.com> * First pass v1 new objects Signed-off-by: Joe Elliott <number101010@gmail.com> * Added failing tail response tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Added failing tailresponse test Signed-off-by: Joe Elliott <number101010@gmail.com> * Partial v1 tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Improved legacy labels test Signed-off-by: Joe Elliott <number101010@gmail.com> * Improved legacy query tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Moved all legacy tests to new method Signed-off-by: Joe Elliott <number101010@gmail.com> * Added v1 tests and fixed stream marshalling bug Signed-off-by: Joe Elliott <number101010@gmail.com> * First pass new Model Signed-off-by: Joe Elliott <number101010@gmail.com> * Added vector test Signed-off-by: Joe Elliott <number101010@gmail.com> * Added matrix tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Added conversions for all things except tailed responses Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed mixed case issues Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed tail marshalling Signed-off-by: Joe Elliott <number101010@gmail.com> * Removed unused testStream Signed-off-by: Joe Elliott <number101010@gmail.com> * Moved TailResponse to loghttp Signed-off-by: Joe Elliott <number101010@gmail.com> * Removed legacy tailresponse objects in favor of actual legacy tailresponse objects Signed-off-by: Joe Elliott <number101010@gmail.com> * Updated v1 methods to take legacy tail objects Signed-off-by: Joe Elliott <number101010@gmail.com> * Cleaned up tests. Added some comments Signed-off-by: Joe Elliott <number101010@gmail.com> * Versioned tail endpoint Signed-off-by: Joe Elliott <number101010@gmail.com> * Improved readability on loghttp packages in http.go Signed-off-by: Joe Elliott <number101010@gmail.com> * Removed new as a var name Signed-off-by: Joe Elliott <number101010@gmail.com> * Started all error messages with lowercase alerts Signed-off-by: Joe Elliott <number101010@gmail.com> * new => ret Signed-off-by: Joe Elliott <number101010@gmail.com> * Added comments on exported methods Signed-off-by: Joe Elliott <number101010@gmail.com> * Removed two personal notes Signed-off-by: Joe Elliott <number101010@gmail.com> * Changed legacy package name to loghttp Signed-off-by: Joe Elliott <number101010@gmail.com> * Moved and renamed loghttp v1 package Signed-off-by: Joe Elliott <number101010@gmail.com> * Moved marshalling code out of model Signed-off-by: Joe Elliott <number101010@gmail.com> * Added package comments Signed-off-by: Joe Elliott <number101010@gmail.com> * Added legacy testing Signed-off-by: Joe Elliott <number101010@gmail.com> * Changed DroppedStream slice to value type for consistency Signed-off-by: Joe Elliott <number101010@gmail.com> * gofmt'ed test files Signed-off-by: Joe Elliott <number101010@gmail.com> * Cleaned up linting issues Signed-off-by: Joe Elliott <number101010@gmail.com> * Minor comment cleanup Signed-off-by: Joe Elliott <number101010@gmail.com> * Adjusted GOGC to make CircleCI happy Signed-off-by: Joe Elliott <number101010@gmail.com> * Changed legacy => loghttp for consistency Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed matrix error message to be correct Signed-off-by: Joe Elliott <number101010@gmail.com> * Moved label query over to loghttp response Signed-off-by: Joe Elliott <number101010@gmail.com> * Added marshal loop tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Added response type test Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed tail response marshal/unmarshal Signed-off-by: Joe Elliott <number101010@gmail.com> * Passing unmarshal/marshal queryresponse tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed vector and matrix Signed-off-by: Joe Elliott <number101010@gmail.com> * Added output support for streams minus ordering Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed tailing Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed output tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Fixed query tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Order log output Signed-off-by: Joe Elliott <number101010@gmail.com> * Use labels instead of stream Signed-off-by: Joe Elliott <number101010@gmail.com> * Lowered parallelization for CircleCI Signed-off-by: Joe Elliott <number101010@gmail.com>
grafana · Sep 23, 2019 · 9614b02 · 9614b02
1 parent 158ca9d
commit 9614b02
Show file tree

Hide file tree

Showing 34 changed files with 1,477 additions and 272 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -78,7 +78,7 @@ jobs:
       - checkout
       - run:
           name: build
-          command: make GOOS=windows promtail
+          command: make GOOS=windows GOGC=10 promtail
 
   # Docker driver
   build/docker-driver:

diff --git a/Makefile b/Makefile
@@ -214,7 +214,7 @@ lint:
 ########
 
 test: all
-	go test -p=6 ./...
+	GOGC=10 go test -p=4 ./...
 
 #########
 # Clean #

diff --git a/pkg/canary/reader/reader.go b/pkg/canary/reader/reader.go
@@ -18,8 +18,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
+	loghttp "github.com/grafana/loki/pkg/loghttp/legacy"
 	"github.com/grafana/loki/pkg/logproto"
-	"github.com/grafana/loki/pkg/querier"
 )
 
 var (
@@ -159,7 +159,7 @@ func (r *Reader) run() {
 
 	r.closeAndReconnect()
 
-	tailResponse := &querier.TailResponse{}
+	tailResponse := &loghttp.TailResponse{}
 
 	for {
 		err := r.conn.ReadJSON(tailResponse)

diff --git a/pkg/logcli/client/client.go b/pkg/logcli/client/client.go
@@ -11,13 +11,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/prometheus/common/model"
-
-	"github.com/grafana/loki/pkg/logql"
+	"github.com/grafana/loki/pkg/loghttp"
 
 	"github.com/gorilla/websocket"
 	"github.com/prometheus/common/config"
-	"github.com/prometheus/prometheus/promql"
 
 	"github.com/grafana/loki/pkg/logproto"
 )
@@ -38,16 +35,10 @@ type Client struct {
 	Address   string
 }
 
-// QueryResult contains fields necessary to return data from Loki endpoints
-type QueryResult struct {
-	ResultType promql.ValueType
-	Result     interface{}
-}
-
 // Query uses the /api/v1/query endpoint to execute an instant query
 // excluding interfacer b/c it suggests taking the interface promql.Node instead of logproto.Direction b/c it happens to have a String() method
 // nolint:interfacer
-func (c *Client) Query(queryStr string, limit int, time time.Time, direction logproto.Direction, quiet bool) (*QueryResult, error) {
+func (c *Client) Query(queryStr string, limit int, time time.Time, direction logproto.Direction, quiet bool) (*loghttp.QueryResponse, error) {
 	path := fmt.Sprintf(queryPath,
 		url.QueryEscape(queryStr), // query
 		limit,                     // limit
@@ -61,7 +52,7 @@ func (c *Client) Query(queryStr string, limit int, time time.Time, direction log
 // QueryRange uses the /api/v1/query_range endpoint to execute a range query
 // excluding interfacer b/c it suggests taking the interface promql.Node instead of logproto.Direction b/c it happens to have a String() method
 // nolint:interfacer
-func (c *Client) QueryRange(queryStr string, limit int, from, through time.Time, direction logproto.Direction, quiet bool) (*QueryResult, error) {
+func (c *Client) QueryRange(queryStr string, limit int, from, through time.Time, direction logproto.Direction, quiet bool) (*loghttp.QueryResponse, error) {
 	path := fmt.Sprintf(queryRangePath,
 		url.QueryEscape(queryStr), // query
 		limit,                     // limit
@@ -74,64 +65,33 @@ func (c *Client) QueryRange(queryStr string, limit int, from, through time.Time,
 }
 
 // ListLabelNames uses the /api/v1/label endpoint to list label names
-func (c *Client) ListLabelNames(quiet bool) (*logproto.LabelResponse, error) {
-	var labelResponse logproto.LabelResponse
+func (c *Client) ListLabelNames(quiet bool) (*loghttp.LabelResponse, error) {
+	var labelResponse loghttp.LabelResponse
 	if err := c.doRequest(labelsPath, quiet, &labelResponse); err != nil {
 		return nil, err
 	}
 	return &labelResponse, nil
 }
 
 // ListLabelValues uses the /api/v1/label endpoint to list label values
-func (c *Client) ListLabelValues(name string, quiet bool) (*logproto.LabelResponse, error) {
+func (c *Client) ListLabelValues(name string, quiet bool) (*loghttp.LabelResponse, error) {
 	path := fmt.Sprintf(labelValuesPath, url.PathEscape(name))
-	var labelResponse logproto.LabelResponse
+	var labelResponse loghttp.LabelResponse
 	if err := c.doRequest(path, quiet, &labelResponse); err != nil {
 		return nil, err
 	}
 	return &labelResponse, nil
 }
 
-func (c *Client) doQuery(path string, quiet bool) (*QueryResult, error) {
+func (c *Client) doQuery(path string, quiet bool) (*loghttp.QueryResponse, error) {
 	var err error
+	var r loghttp.QueryResponse
 
-	unmarshal := struct {
-		Type   promql.ValueType `json:"resultType"`
-		Result json.RawMessage  `json:"result"`
-	}{}
-
-	if err = c.doRequest(path, quiet, &unmarshal); err != nil {
-		return nil, err
-	}
-
-	var value interface{}
-
-	// unmarshal results
-	switch unmarshal.Type {
-	case logql.ValueTypeStreams:
-		var s logql.Streams
-		err = json.Unmarshal(unmarshal.Result, &s)
-		value = s
-	case promql.ValueTypeMatrix:
-		var m model.Matrix
-		err = json.Unmarshal(unmarshal.Result, &m)
-		value = m
-	case promql.ValueTypeVector:
-		var m model.Vector
-		err = json.Unmarshal(unmarshal.Result, &m)
-		value = m
-	default:
-		return nil, fmt.Errorf("Unknown type: %s", unmarshal.Type)
-	}
-
-	if err != nil {
+	if err = c.doRequest(path, quiet, &r); err != nil {
 		return nil, err
 	}
 
-	return &QueryResult{
-		ResultType: unmarshal.Type,
-		Result:     value,
-	}, nil
+	return &r, nil
 }
 
 func (c *Client) doRequest(path string, quiet bool, out interface{}) error {

diff --git a/pkg/logcli/labelquery/labels.go b/pkg/logcli/labelquery/labels.go
@@ -5,7 +5,7 @@ import (
 	"log"
 
 	"github.com/grafana/loki/pkg/logcli/client"
-	"github.com/grafana/loki/pkg/logproto"
+	"github.com/grafana/loki/pkg/loghttp"
 )
 
 // LabelQuery contains all necessary fields to execute label queries and print out the resutls
@@ -25,7 +25,7 @@ func (q *LabelQuery) DoLabels(c *client.Client) {
 
 // ListLabels returns an array of label strings
 func (q *LabelQuery) ListLabels(c *client.Client) []string {
-	var labelResponse *logproto.LabelResponse
+	var labelResponse *loghttp.LabelResponse
 	var err error
 	if len(q.LabelName) > 0 {
 		labelResponse, err = c.ListLabelValues(q.LabelName, q.Quiet)

diff --git a/pkg/logcli/output/default.go b/pkg/logcli/output/default.go
@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/fatih/color"
-	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/grafana/loki/pkg/loghttp"
 )
 
 // DefaultOutput provides logs and metadata in human readable format
@@ -15,19 +15,19 @@ type DefaultOutput struct {
 }
 
 // Format a log entry in a human readable format
-func (o *DefaultOutput) Format(ts time.Time, lbls *labels.Labels, maxLabelsLen int, line string) string {
+func (o *DefaultOutput) Format(ts time.Time, lbls loghttp.LabelSet, maxLabelsLen int, line string) string {
 	timestamp := ts.In(o.options.Timezone).Format(time.RFC3339)
 	line = strings.TrimSpace(line)
 
 	if o.options.NoLabels {
 		return fmt.Sprintf("%s %s", color.BlueString(timestamp), line)
 	}
 
-	return fmt.Sprintf("%s %s %s", color.BlueString(timestamp), color.RedString(padLabel(*lbls, maxLabelsLen)), line)
+	return fmt.Sprintf("%s %s %s", color.BlueString(timestamp), color.RedString(padLabel(lbls, maxLabelsLen)), line)
 }
 
 // add some padding after labels
-func padLabel(ls labels.Labels, maxLabelsLen int) string {
+func padLabel(ls loghttp.LabelSet, maxLabelsLen int) string {
 	labels := ls.String()
 	if len(labels) < maxLabelsLen {
 		labels += strings.Repeat(" ", maxLabelsLen-len(labels))

diff --git a/pkg/logcli/output/default_test.go b/pkg/logcli/output/default_test.go
@@ -5,69 +5,71 @@ import (
 	"testing"
 	"time"
 
-	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/grafana/loki/pkg/loghttp"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestDefaultOutput_Format(t *testing.T) {
 	t.Parallel()
 
 	timestamp, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05+07:00")
-	emptyLabels := labels.New()
-	someLabels := labels.New(labels.Label{Name: "type", Value: "test"})
+	emptyLabels := loghttp.LabelSet{}
+	someLabels := loghttp.LabelSet(map[string]string{
+		"type": "test",
+	})
 
 	tests := map[string]struct {
 		options      *LogOutputOptions
 		timestamp    time.Time
-		lbls         *labels.Labels
+		lbls         loghttp.LabelSet
 		maxLabelsLen int
 		line         string
 		expected     string
 	}{
 		"empty line with no labels": {
 			&LogOutputOptions{Timezone: time.UTC, NoLabels: false},
 			timestamp,
-			&emptyLabels,
+			emptyLabels,
 			0,
 			"",
 			"2006-01-02T08:04:05Z {} ",
 		},
 		"empty line with labels": {
 			&LogOutputOptions{Timezone: time.UTC, NoLabels: false},
 			timestamp,
-			&someLabels,
+			someLabels,
 			len(someLabels.String()),
 			"",
 			"2006-01-02T08:04:05Z {type=\"test\"} ",
 		},
 		"max labels length shorter than input labels": {
 			&LogOutputOptions{Timezone: time.UTC, NoLabels: false},
 			timestamp,
-			&someLabels,
+			someLabels,
 			0,
 			"Hello",
 			"2006-01-02T08:04:05Z {type=\"test\"} Hello",
 		},
 		"max labels length longer than input labels": {
 			&LogOutputOptions{Timezone: time.UTC, NoLabels: false},
 			timestamp,
-			&someLabels,
+			someLabels,
 			20,
 			"Hello",
 			"2006-01-02T08:04:05Z {type=\"test\"}        Hello",
 		},
 		"timezone option set to a Local one": {
 			&LogOutputOptions{Timezone: time.FixedZone("test", 2*60*60), NoLabels: false},
 			timestamp,
-			&someLabels,
+			someLabels,
 			0,
 			"Hello",
 			"2006-01-02T10:04:05+02:00 {type=\"test\"} Hello",
 		},
 		"labels output disabled": {
 			&LogOutputOptions{Timezone: time.UTC, NoLabels: true},
 			timestamp,
-			&someLabels,
+			someLabels,
 			0,
 			"Hello",
 			"2006-01-02T08:04:05Z Hello",
@@ -92,10 +94,17 @@ func TestDefaultOutput_FormatLabelsPadding(t *testing.T) {
 	t.Parallel()
 
 	// Define a list of labels that - once formatted - have a different length
-	labelsList := []labels.Labels{
-		labels.New(labels.Label{Name: "type", Value: "test"}),
-		labels.New(labels.Label{Name: "type", Value: "test"}, labels.Label{Name: "foo", Value: "bar"}),
-		labels.New(labels.Label{Name: "type", Value: "a-longer-test"}),
+	labelsList := []loghttp.LabelSet{
+		loghttp.LabelSet(map[string]string{
+			"type": "test",
+		}),
+		loghttp.LabelSet(map[string]string{
+			"type": "test",
+			"foo":  "bar",
+		}),
+		loghttp.LabelSet(map[string]string{
+			"type": "a-longer-test",
+		}),
 	}
 
 	timestamp, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05+07:00")
@@ -106,7 +115,7 @@ func TestDefaultOutput_FormatLabelsPadding(t *testing.T) {
 	// Format the same log line with different labels
 	formattedEntries := make([]string, 0, len(labelsList))
 	for _, lbls := range labelsList {
-		formattedEntries = append(formattedEntries, out.Format(timestamp, &lbls, maxLabelsLen, "XXX"))
+		formattedEntries = append(formattedEntries, out.Format(timestamp, lbls, maxLabelsLen, "XXX"))
 	}
 
 	// Ensure the log line starts at the same position in each formatted output
@@ -122,7 +131,7 @@ func TestDefaultOutput_FormatLabelsPadding(t *testing.T) {
 	}
 }
 
-func findMaxLabelsLength(labelsList []labels.Labels) int {
+func findMaxLabelsLength(labelsList []loghttp.LabelSet) int {
 	maxLabelsLen := 0
 
 	for _, lbls := range labelsList {

diff --git a/pkg/logcli/output/jsonl.go b/pkg/logcli/output/jsonl.go
@@ -5,7 +5,7 @@ import (
 	"log"
 	"time"
 
-	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/grafana/loki/pkg/loghttp"
 )
 
 // JSONLOutput prints logs and metadata as JSON Lines, suitable for scripts
@@ -14,7 +14,7 @@ type JSONLOutput struct {
 }
 
 // Format a log entry as json line
-func (o *JSONLOutput) Format(ts time.Time, lbls *labels.Labels, maxLabelsLen int, line string) string {
+func (o *JSONLOutput) Format(ts time.Time, lbls loghttp.LabelSet, maxLabelsLen int, line string) string {
 	entry := map[string]interface{}{
 		"timestamp": ts.In(o.options.Timezone),
 		"line":      line,