Move chunks-inspect tool to Loki repo

cmd/chunks-inspect/.gitignore

@@ -0,0 +1 @@
+chunks-inspect

cmd/chunks-inspect/README.md

@@ -0,0 +1,69 @@
+# Tools for inspecting Loki chunks
+
+This tool can parse Loki chunks and print details from them. Useful for Loki developers.
+
+To build the tool, simply run `go build` in this directory. Running resulting program with chunks file name gives you some basic chunks information:
+
+```shell
+$ ./chunks-inspect db61b4eca2a5ad68\:16f89ff4164\:16f8a0cfb41\:1538ace0 
+
+Chunks file: db61b4eca2a5ad68:16f89ff4164:16f8a0cfb41:1538ace0
+Metadata length: 485
+Data length: 264737
+UserID: 29
+From: 2020-01-09 11:10:04.644000 UTC
+Through: 2020-01-09 11:25:04.193000 UTC (14m59.549s)
+Labels:
+	 __name__ = logs
+	 app = graphite
+	 cluster = us-central1
+	 color = b
+	 container_name = graphite
+	 filename = /var/log/pods/metrictank_graphite-1-large-multitenant-b-5f9db68b5c-jh769_ca9a10b0-0d2d-11ea-b85a-42010a80017a/graphite/0.log
+	 hosted_metrics = 1
+	 instance = graphite-1-large-multitenant-b-5f9db68b5c-jh769
+	 job = metrictank/graphite
+	 namespace = metrictank
+	 org = 1
+	 plan = large
+	 pod_template_hash = 5f9db68b5c
+	 stream = stderr
+Encoding: lz4
+Blocks Metadata Checksum: 3444d7a3 OK
+Found 5 block(s), use -b to show block details
+Minimum time (from first block): 2020-01-09 11:10:04.644490 UTC
+Maximum time (from last block): 2020-01-09 11:25:04.192368 UTC
+Total size of original data: 1257319 file size: 265226 ratio: 4.74
+```
+
+To print more details about individual blocks inside chunks, use `-b` parameter:
+
+```shell script
+$ ./chunks-inspect -b db61b4eca2a5ad68\:16f89ff4164\:16f8a0cfb41\:1538ace0
+
+... chunk file info, see above ...
+
+Block    0: position:        6, original length: 273604 (stored:  56220, ratio: 4.87), minT: 2020-01-09 11:10:04.644490 UTC maxT: 2020-01-09 11:12:53.458289 UTC, checksum: 13e73d71 OK
+Block    0: digest compressed: ae657fdbb2b8be55eebe86b31a21050de2b5e568444507e5958218710ddf02fd, original: 0dad619bf3049a1152cb3153d90c6db6c3f54edbf9977753dde3c4e1b09d07b4
+Block    1: position:    56230, original length: 274703 (stored:  60861, ratio: 4.51), minT: 2020-01-09 11:12:53.461855 UTC maxT: 2020-01-09 11:16:35.420787 UTC, checksum: 55269e65 OK
+Block    1: digest compressed: a7999f471f68cce0458ff9790e7e7501c5bfe14cc28661d8670b9d88aeaee96f, original: a617a9e0b6c33aeaa83833470cf6164c540a7a64258e55eec6fdff483059df6f
+Block    2: position:   117095, original length: 273592 (stored:  56563, ratio: 4.84), minT: 2020-01-09 11:16:35.423228 UTC maxT: 2020-01-09 11:19:28.680048 UTC, checksum: 781dba21 OK
+Block    2: digest compressed: 65b59cc61c5eeea8116ce8a8c0b0d98b4d4671e8bc91656979c93717050a18fc, original: 896cc6487365ad0590097794a202aad5c89776d1c626f2cea33c652885939ac6
+Block    3: position:   173662, original length: 273745 (stored:  57486, ratio: 4.76), minT: 2020-01-09 11:19:31.062836 UTC maxT: 2020-01-09 11:23:13.562630 UTC, checksum: 2a88a52b OK
+Block    3: digest compressed: 4f51a64d0397cc806a898cd6662695620083466f234d179fef5c2d02c9766191, original: 15e8a1833ccbba9aa8374029141a054127526382423d3a63f321698ff8e087b5
+Block    4: position:   231152, original length: 161675 (stored:  33440, ratio: 4.83), minT: 2020-01-09 11:23:15.416284 UTC maxT: 2020-01-09 11:25:04.192368 UTC, checksum: 6d952296 OK
+Block    4: digest compressed: 8dd12235f1d619c30a9afb66823a6c827613257773669fda6fbfe014ed623cd1, original: 1f7e8ef8eb937c87ad3ed3e24c321c40d43534cc43662f83ab493fb3391548b2
+Total size of original data: 1257319 file size: 265226 ratio: 4.74
+```
+
+To also print individual log lines, use `-l` parameter. Full help:
+
+```shell script
+$ ./chunks-inspect -h
+Usage of ./chunks-inspect:
+  -b	print block details
+  -l	print log lines
+  -s	store blocks, using input filename, and appending block index to it
+```
+
+Parameter `-s` allows you to inspect individual blocks, both in compressed format (as stored in chunk file), and original raw format.

cmd/chunks-inspect/go.mod

@@ -0,0 +1,9 @@
+module github.com/grafana/loki/cmd/chunks-inspect
+
+go 1.13
+
+require (
+	github.com/frankban/quicktest v1.7.2 // indirect
+	github.com/golang/snappy v0.0.1
+	github.com/pierrec/lz4 v2.3.0+incompatible
+)

cmd/chunks-inspect/go.sum

@@ -0,0 +1,13 @@
+github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
+github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pierrec/lz4 v2.3.0+incompatible h1:CZzRn4Ut9GbUkHlQ7jqBXeZQV41ZSKWFc302ZU6lUTk=
+github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

cmd/chunks-inspect/header.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/golang/snappy"
+)
+
+type ChunkHeader struct {
+	// These two fields will be missing from older chunks (as will the hash).
+	// On fetch we will initialise these fields from the DynamoDB key.
+	Fingerprint uint64 `json:"fingerprint"` // model.Fingerprint
+	UserID      string `json:"userID"`
+
+	// These fields will be in all chunks, including old ones.
+	From    Time   `json:"from"`    // model.Time
+	Through Time   `json:"through"` // model.Time
+	Metric  Labels `json:"metric"`
+
+	// We never use Delta encoding (the zero value), so if this entry is
+	// missing, we default to DoubleDelta.
+	Encoding byte `json:"encoding"`
+
+	MetadataLength uint32
+	DataLength     uint32
+}
+
+// Decode the chunk from the given buffer, and confirm the chunk is the one we
+// expected.
+func DecodeHeader(r io.Reader) (*ChunkHeader, error) {
+	// Now unmarshal the chunk metadata.
+	var metadataLen uint32
+	if err := binary.Read(r, binary.BigEndian, &metadataLen); err != nil {
+		return nil, fmt.Errorf("when reading metadata length from chunk: %w", err)
+	}
+
+	metadataBytes := make([]byte, metadataLen-4) // writer includes size of "metadataLen" in the length
+	if _, err := io.ReadFull(r, metadataBytes); err != nil {
+		return nil, fmt.Errorf("error while reading metadata bytes: %w", err)
+	}
+
+	var metadata ChunkHeader
+	if err := json.NewDecoder(snappy.NewReader(bytes.NewReader(metadataBytes))).Decode(&metadata); err != nil {
+		return nil, fmt.Errorf("error while decoding metadata: %w", err)
+	}
+
+	var dataLen uint32
+	if err := binary.Read(r, binary.BigEndian, &dataLen); err != nil {
+		return nil, fmt.Errorf("when reading rawData length from chunk: %w", err)
+	}
+
+	metadata.MetadataLength = metadataLen
+	metadata.DataLength = dataLen
+
+	return &metadata, nil
+}

cmd/chunks-inspect/labels.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"sort"
+	"strconv"
+)
+
+type Label struct {
+	Name, Value string
+}
+
+type Labels []Label
+
+func (ls Labels) Len() int           { return len(ls) }
+func (ls Labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
+func (ls Labels) Less(i, j int) bool { return ls[i].Name < ls[j].Name }
+
+func (ls Labels) String() string {
+	var b bytes.Buffer
+
+	b.WriteByte('{')
+	for i, l := range ls {
+		if i > 0 {
+			b.WriteByte(',')
+			b.WriteByte(' ')
+		}
+		b.WriteString(l.Name)
+		b.WriteByte('=')
+		b.WriteString(strconv.Quote(l.Value))
+	}
+	b.WriteByte('}')
+
+	return b.String()
+}
+
+// FromMap returns new sorted Labels from the given map.
+func FromMap(m map[string]string) Labels {
+	l := make([]Label, 0, len(m))
+	for k, v := range m {
+		l = append(l, Label{Name: k, Value: v})
+	}
+	return New(l...)
+}
+
+// New returns a sorted Labels from the given labels.
+// The caller has to guarantee that all label names are unique.
+func New(ls ...Label) Labels {
+	set := make(Labels, 0, len(ls))
+	for _, l := range ls {
+		set = append(set, l)
+	}
+	sort.Sort(set)
+
+	return set
+}
+
+// Map returns a string map of the labels.
+func (ls Labels) Map() map[string]string {
+	m := make(map[string]string, len(ls))
+	for _, l := range ls {
+		m[l.Name] = l.Value
+	}
+	return m
+}
+
+// MarshalJSON implements json.Marshaler.
+func (ls Labels) MarshalJSON() ([]byte, error) {
+	return json.Marshal(ls.Map())
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (ls *Labels) UnmarshalJSON(b []byte) error {
+	var m map[string]string
+
+	if err := json.Unmarshal(b, &m); err != nil {
+		return err
+	}
+
+	*ls = FromMap(m)
+	return nil
+}