From 5916cc2b4733a185fef60640736f13b676a327b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Fri, 24 Jan 2020 16:14:54 +0100 Subject: [PATCH 1/3] Move chunks-inspect tool to Loki repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- cmd/chunks-inspect/go.mod | 9 ++ cmd/chunks-inspect/go.sum | 13 +++ cmd/chunks-inspect/header.go | 60 ++++++++++ cmd/chunks-inspect/labels.go | 83 ++++++++++++++ cmd/chunks-inspect/loki.go | 213 +++++++++++++++++++++++++++++++++++ cmd/chunks-inspect/main.go | 131 +++++++++++++++++++++ cmd/chunks-inspect/time.go | 79 +++++++++++++ 7 files changed, 588 insertions(+) create mode 100644 cmd/chunks-inspect/go.mod create mode 100644 cmd/chunks-inspect/go.sum create mode 100644 cmd/chunks-inspect/header.go create mode 100644 cmd/chunks-inspect/labels.go create mode 100644 cmd/chunks-inspect/loki.go create mode 100644 cmd/chunks-inspect/main.go create mode 100644 cmd/chunks-inspect/time.go diff --git a/cmd/chunks-inspect/go.mod b/cmd/chunks-inspect/go.mod new file mode 100644 index 000000000000..7ffd87fbcb69 --- /dev/null +++ b/cmd/chunks-inspect/go.mod @@ -0,0 +1,9 @@ +module github.com/grafana/loki/cmd/chunks-inspect + +go 1.13 + +require ( + github.com/frankban/quicktest v1.7.2 // indirect + github.com/golang/snappy v0.0.1 + github.com/pierrec/lz4 v2.3.0+incompatible +) diff --git a/cmd/chunks-inspect/go.sum b/cmd/chunks-inspect/go.sum new file mode 100644 index 000000000000..ce681b5bc1fd --- /dev/null +++ b/cmd/chunks-inspect/go.sum @@ -0,0 +1,13 @@ +github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pierrec/lz4 v2.3.0+incompatible h1:CZzRn4Ut9GbUkHlQ7jqBXeZQV41ZSKWFc302ZU6lUTk= +github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= diff --git a/cmd/chunks-inspect/header.go b/cmd/chunks-inspect/header.go new file mode 100644 index 000000000000..2ea4fd00604f --- /dev/null +++ b/cmd/chunks-inspect/header.go @@ -0,0 +1,60 @@ +package main + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "io" + + "github.com/golang/snappy" +) + +type ChunkHeader struct { + // These two fields will be missing from older chunks (as will the hash). + // On fetch we will initialise these fields from the DynamoDB key. + Fingerprint uint64 `json:"fingerprint"` // model.Fingerprint + UserID string `json:"userID"` + + // These fields will be in all chunks, including old ones. + From Time `json:"from"` // model.Time + Through Time `json:"through"` // model.Time + Metric Labels `json:"metric"` + + // We never use Delta encoding (the zero value), so if this entry is + // missing, we default to DoubleDelta. + Encoding byte `json:"encoding"` + + MetadataLength uint32 + DataLength uint32 +} + +// Decode the chunk from the given buffer, and confirm the chunk is the one we +// expected. +func DecodeHeader(r io.Reader) (*ChunkHeader, error) { + // Now unmarshal the chunk metadata. + var metadataLen uint32 + if err := binary.Read(r, binary.BigEndian, &metadataLen); err != nil { + return nil, fmt.Errorf("when reading metadata length from chunk: %w", err) + } + + metadataBytes := make([]byte, metadataLen-4) // writer includes size of "metadataLen" in the length + if _, err := io.ReadFull(r, metadataBytes); err != nil { + return nil, fmt.Errorf("error while reading metadata bytes: %w", err) + } + + var metadata ChunkHeader + if err := json.NewDecoder(snappy.NewReader(bytes.NewReader(metadataBytes))).Decode(&metadata); err != nil { + return nil, fmt.Errorf("error while decoding metadata: %w", err) + } + + var dataLen uint32 + if err := binary.Read(r, binary.BigEndian, &dataLen); err != nil { + return nil, fmt.Errorf("when reading rawData length from chunk: %w", err) + } + + metadata.MetadataLength = metadataLen + metadata.DataLength = dataLen + + return &metadata, nil +} diff --git a/cmd/chunks-inspect/labels.go b/cmd/chunks-inspect/labels.go new file mode 100644 index 000000000000..9b02a87de4c6 --- /dev/null +++ b/cmd/chunks-inspect/labels.go @@ -0,0 +1,83 @@ +package main + +import ( + "bytes" + "encoding/json" + "sort" + "strconv" +) + +type Label struct { + Name, Value string +} + +type Labels []Label + +func (ls Labels) Len() int { return len(ls) } +func (ls Labels) Swap(i, j int) { ls[i], ls[j] = ls[j], ls[i] } +func (ls Labels) Less(i, j int) bool { return ls[i].Name < ls[j].Name } + +func (ls Labels) String() string { + var b bytes.Buffer + + b.WriteByte('{') + for i, l := range ls { + if i > 0 { + b.WriteByte(',') + b.WriteByte(' ') + } + b.WriteString(l.Name) + b.WriteByte('=') + b.WriteString(strconv.Quote(l.Value)) + } + b.WriteByte('}') + + return b.String() +} + +// FromMap returns new sorted Labels from the given map. +func FromMap(m map[string]string) Labels { + l := make([]Label, 0, len(m)) + for k, v := range m { + l = append(l, Label{Name: k, Value: v}) + } + return New(l...) +} + +// New returns a sorted Labels from the given labels. +// The caller has to guarantee that all label names are unique. +func New(ls ...Label) Labels { + set := make(Labels, 0, len(ls)) + for _, l := range ls { + set = append(set, l) + } + sort.Sort(set) + + return set +} + +// Map returns a string map of the labels. +func (ls Labels) Map() map[string]string { + m := make(map[string]string, len(ls)) + for _, l := range ls { + m[l.Name] = l.Value + } + return m +} + +// MarshalJSON implements json.Marshaler. +func (ls Labels) MarshalJSON() ([]byte, error) { + return json.Marshal(ls.Map()) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (ls *Labels) UnmarshalJSON(b []byte) error { + var m map[string]string + + if err := json.Unmarshal(b, &m); err != nil { + return err + } + + *ls = FromMap(m) + return nil +} diff --git a/cmd/chunks-inspect/loki.go b/cmd/chunks-inspect/loki.go new file mode 100644 index 000000000000..79c1c8cfe407 --- /dev/null +++ b/cmd/chunks-inspect/loki.go @@ -0,0 +1,213 @@ +package main + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + + "github.com/golang/snappy" + "github.com/pierrec/lz4" +) + +type Encoding struct { + code int + name string + readerFn func(io.Reader) (io.Reader, error) +} + +func (e Encoding) String() string { + return e.name +} + +// The table gets initialized with sync.Once but may still cause a race +// with any other use of the crc32 package anywhere. Thus we initialize it +// before. +var castagnoliTable *crc32.Table + +func init() { + castagnoliTable = crc32.MakeTable(crc32.Castagnoli) +} + +var ( + encNone = Encoding{code: 0, name: "none", readerFn: func(reader io.Reader) (io.Reader, error) { return reader, nil }} + encGZIP = Encoding{code: 1, name: "gzip", readerFn: func(reader io.Reader) (io.Reader, error) { return gzip.NewReader(reader) }} + encDumb = Encoding{code: 2, name: "dumb", readerFn: func(reader io.Reader) (io.Reader, error) { return reader, nil }} + encLZ4 = Encoding{code: 3, name: "lz4", readerFn: func(reader io.Reader) (io.Reader, error) { return lz4.NewReader(reader), nil }} + encSnappy = Encoding{code: 4, name: "snappy", readerFn: func(reader io.Reader) (io.Reader, error) { return snappy.NewReader(reader), nil }} + + Encodings = []Encoding{encNone, encGZIP, encDumb, encLZ4, encSnappy} +) + +type LokiChunk struct { + encoding Encoding + + blocks []LokiBlock + + metadataChecksum uint32 + computedMetadataChecksum uint32 +} + +type LokiBlock struct { + numEntries uint64 // number of log lines in this block + minT int64 // minimum timestamp, unix nanoseconds + maxT int64 // max timestamp, unix nanoseconds + + dataOffset uint64 // offset in the data-part of chunks file + + rawData []byte // data as stored in chunk file, compressed + originalData []byte // data uncompressed from rawData + + // parsed rawData + entries []LokiEntry + storedChecksum uint32 + computedChecksum uint32 +} + +type LokiEntry struct { + timestamp int64 + line string +} + +func parseLokiChunk(chunkHeader *ChunkHeader, r io.Reader) (*LokiChunk, error) { + // Loki chunks need to be loaded into memory, because some offsets are actually stored at the end. + data := make([]byte, chunkHeader.DataLength) + if _, err := io.ReadFull(r, data); err != nil { + return nil, fmt.Errorf("failed to read rawData for Loki chunk into memory: %w", err) + } + + if num := binary.BigEndian.Uint32(data[0:4]); num != 0x012EE56A { + return nil, fmt.Errorf("invalid magic number: %0x", num) + } + + compression, err := getCompression(data[4], data[5]) + if err != nil { + return nil, fmt.Errorf("failed to read compression: %w", err) + } + + // return &LokiChunk{encoding: compression}, nil + + metasOffset := binary.BigEndian.Uint64(data[len(data)-8:]) + + metadata := data[metasOffset : len(data)-(8+4)] + + metaChecksum := binary.BigEndian.Uint32(data[len(data)-12 : len(data)-8]) + computedMetaChecksum := crc32.Checksum(metadata, castagnoliTable) + + blocks, n := binary.Uvarint(metadata) + if n <= 0 { + return nil, fmt.Errorf("failed to read number of blocks") + } + metadata = metadata[n:] + + lokiChunk := &LokiChunk{ + encoding: compression, + metadataChecksum: metaChecksum, + computedMetadataChecksum: computedMetaChecksum, + } + + for ix := 0; ix < int(blocks); ix++ { + block := LokiBlock{} + block.numEntries, metadata, err = readUvarint(err, metadata) + block.minT, metadata, err = readVarint(err, metadata) + block.maxT, metadata, err = readVarint(err, metadata) + block.dataOffset, metadata, err = readUvarint(err, metadata) + dataLength := uint64(0) + dataLength, metadata, err = readUvarint(err, metadata) + + if err != nil { + return nil, err + } + + block.rawData = data[block.dataOffset : block.dataOffset+dataLength] + block.storedChecksum = binary.BigEndian.Uint32(data[block.dataOffset+dataLength : block.dataOffset+dataLength+4]) + block.computedChecksum = crc32.Checksum(block.rawData, castagnoliTable) + block.originalData, block.entries, err = parseLokiBlock(compression, block.rawData) + lokiChunk.blocks = append(lokiChunk.blocks, block) + } + + return lokiChunk, nil +} + +func parseLokiBlock(compression Encoding, data []byte) ([]byte, []LokiEntry, error) { + r, err := compression.readerFn(bytes.NewReader(data)) + if err != nil { + return nil, nil, err + } + + decompressed, err := ioutil.ReadAll(r) + origDecompressed := decompressed + if err != nil { + return nil, nil, err + } + + entries := []LokiEntry(nil) + for len(decompressed) > 0 { + var timestamp int64 + var lineLength uint64 + + timestamp, decompressed, err = readVarint(err, decompressed) + lineLength, decompressed, err = readUvarint(err, decompressed) + if err != nil { + return origDecompressed, nil, err + } + + if len(decompressed) < int(lineLength) { + return origDecompressed, nil, fmt.Errorf("not enough line data, need %d, got %d", lineLength, len(decompressed)) + } + + entries = append(entries, LokiEntry{ + timestamp: timestamp, + line: string(decompressed[0:lineLength]), + }) + + decompressed = decompressed[lineLength:] + } + + return origDecompressed, entries, nil +} + +func readVarint(prevErr error, buf []byte) (int64, []byte, error) { + if prevErr != nil { + return 0, buf, prevErr + } + + val, n := binary.Varint(buf) + if n <= 0 { + return 0, nil, fmt.Errorf("varint: %d", n) + } + return val, buf[n:], nil +} + +func readUvarint(prevErr error, buf []byte) (uint64, []byte, error) { + if prevErr != nil { + return 0, buf, prevErr + } + + val, n := binary.Uvarint(buf) + if n <= 0 { + return 0, nil, fmt.Errorf("varint: %d", n) + } + return val, buf[n:], nil +} + +func getCompression(format byte, code byte) (Encoding, error) { + if format == 1 { + return encGZIP, nil + } + + if format == 2 { + for _, e := range Encodings { + if e.code == int(code) { + return e, nil + } + } + + return encNone, fmt.Errorf("unknown encoding: %d", code) + } + + return encNone, fmt.Errorf("unknown format: %d", format) +} diff --git a/cmd/chunks-inspect/main.go b/cmd/chunks-inspect/main.go new file mode 100644 index 000000000000..f88193e2b4ed --- /dev/null +++ b/cmd/chunks-inspect/main.go @@ -0,0 +1,131 @@ +package main + +import ( + "crypto/sha256" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "time" +) + +const format = "2006-01-02 15:04:05.000000 MST" + +var timezone = time.UTC + +func main() { + blocks := flag.Bool("b", false, "print block details") + lines := flag.Bool("l", false, "print log lines") + storeBlocks := flag.Bool("s", false, "store blocks, using input filename, and appending block index to it") + flag.Parse() + + for _, f := range flag.Args() { + printFile(f, *blocks, *lines, *storeBlocks) + } +} + +func printFile(filename string, blockDetails, printLines, storeBlocks bool) { + f, err := os.Open(filename) + if err != nil { + log.Printf("%s: %v", filename, err) + return + } + defer f.Close() + + si, err := f.Stat() + if err != nil { + log.Println("failed to stat file", err) + return + } + + h, err := DecodeHeader(f) + if err != nil { + log.Printf("%s: %v", filename, err) + return + } + + fmt.Println() + fmt.Println("Chunks file:", filename) + fmt.Println("Metadata length:", h.MetadataLength) + fmt.Println("Data length:", h.DataLength) + fmt.Println("UserID:", h.UserID) + from, through := h.From.Time().In(timezone), h.Through.Time().In(timezone) + fmt.Println("From:", from.Format(format)) + fmt.Println("Through:", through.Format(format), "("+through.Sub(from).String()+")") + fmt.Println("Labels:") + + for _, l := range h.Metric { + fmt.Println("\t", l.Name, "=", l.Value) + } + + lokiChunk, err := parseLokiChunk(h, f) + if err != nil { + log.Printf("%s: %v", filename, err) + return + } + + fmt.Println("Encoding:", lokiChunk.encoding) + fmt.Print("Blocks Metadata Checksum: ", fmt.Sprintf("%08x", lokiChunk.metadataChecksum)) + if lokiChunk.metadataChecksum == lokiChunk.computedMetadataChecksum { + fmt.Println(" OK") + } else { + fmt.Println(" BAD, computed checksum:", fmt.Sprintf("%08x", lokiChunk.computedMetadataChecksum)) + } + if blockDetails { + fmt.Println("Found", len(lokiChunk.blocks), "block(s)") + } else { + fmt.Println("Found", len(lokiChunk.blocks), "block(s), use -b to show block details") + } + if len(lokiChunk.blocks) > 0 { + fmt.Println("Minimum time (from first block):", time.Unix(0, lokiChunk.blocks[0].minT).In(timezone).Format(format)) + fmt.Println("Maximum time (from last block):", time.Unix(0, lokiChunk.blocks[len(lokiChunk.blocks) - 1].maxT).In(timezone).Format(format)) + } + + if blockDetails { + fmt.Println() + } + + totalSize := 0 + + for ix, b := range lokiChunk.blocks { + if blockDetails { + cksum := "" + if b.storedChecksum == b.computedChecksum { + cksum = fmt.Sprintf("%08x OK", b.storedChecksum) + } else { + cksum = fmt.Sprintf("%08x BAD (computed: %08x)", b.storedChecksum, b.computedChecksum) + } + fmt.Printf("Block %4d: position: %8d, original length: %6d (stored: %6d, ratio: %.2f), minT: %v maxT: %v, checksum: %s\n", + ix, b.dataOffset, len(b.originalData), len(b.rawData), float64(len(b.originalData))/float64(len(b.rawData)), + time.Unix(0, b.minT).In(timezone).Format(format), time.Unix(0, b.maxT).In(timezone).Format(format), + cksum) + fmt.Printf("Block %4d: digest compressed: %02x, original: %02x\n", ix, sha256.Sum256(b.rawData), sha256.Sum256(b.originalData)) + } + + totalSize += len(b.originalData) + + if printLines { + for _, l := range b.entries { + fmt.Printf("%v\t%s\n", time.Unix(0, l.timestamp).In(timezone).Format(format), strings.TrimSpace(l.line)) + } + } + + if storeBlocks { + writeBlockToFile(b.rawData, ix, fmt.Sprintf("%s.block.%d", filename, ix)) + writeBlockToFile(b.originalData, ix, fmt.Sprintf("%s.original.%d", filename, ix)) + } + } + + fmt.Println("Total size of original data:", totalSize, "file size:", si.Size(), "ratio:", fmt.Sprintf("%0.3g", float64(totalSize)/float64(si.Size()))) +} + +func writeBlockToFile(data []byte, blockIndex int, filename string) { + err := ioutil.WriteFile(filename, data, 0644) + if err != nil { + log.Println("Failed to store block", blockIndex, "to file", filename, "due to error:", err) + } else { + log.Println("Stored block", blockIndex, "to file", filename) + } +} diff --git a/cmd/chunks-inspect/time.go b/cmd/chunks-inspect/time.go new file mode 100644 index 000000000000..aac421cc8912 --- /dev/null +++ b/cmd/chunks-inspect/time.go @@ -0,0 +1,79 @@ +package main + +import ( + "fmt" + "math" + "strconv" + "strings" + "time" +) + +const ( + minimumTick = time.Millisecond + second = int64(time.Second / minimumTick) + nanosPerTick = int64(minimumTick / time.Nanosecond) +) + +// The number of digits after the dot. +var dotPrecision = int(math.Log10(float64(second))) + +type Time int64 + +// String returns a string representation of the Time. +func (t Time) String() string { + return strconv.FormatFloat(float64(t)/float64(second), 'f', -1, 64) +} + +// MarshalJSON implements the json.Marshaler interface. +func (t Time) MarshalJSON() ([]byte, error) { + return []byte(t.String()), nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (t *Time) UnmarshalJSON(b []byte) error { + p := strings.Split(string(b), ".") + switch len(p) { + case 1: + v, err := strconv.ParseInt(string(p[0]), 10, 64) + if err != nil { + return err + } + *t = Time(v * second) + + case 2: + v, err := strconv.ParseInt(string(p[0]), 10, 64) + if err != nil { + return err + } + v *= second + + prec := dotPrecision - len(p[1]) + if prec < 0 { + p[1] = p[1][:dotPrecision] + } else if prec > 0 { + p[1] = p[1] + strings.Repeat("0", prec) + } + + va, err := strconv.ParseInt(p[1], 10, 32) + if err != nil { + return err + } + + // If the value was something like -0.1 the negative is lost in the + // parsing because of the leading zero, this ensures that we capture it. + if len(p[0]) > 0 && p[0][0] == '-' && v+va > 0 { + *t = Time(v+va) * -1 + } else { + *t = Time(v + va) + } + + default: + return fmt.Errorf("invalid time %q", string(b)) + } + return nil +} + +// Time returns the time.Time representation of t. +func (t Time) Time() time.Time { + return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick) +} From 6065ff0afc15fbe263167929dd4c3e11ccd60189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Fri, 24 Jan 2020 16:16:58 +0100 Subject: [PATCH 2/3] Added README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- cmd/chunks-inspect/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 cmd/chunks-inspect/README.md diff --git a/cmd/chunks-inspect/README.md b/cmd/chunks-inspect/README.md new file mode 100644 index 000000000000..7dbe878b2b2a --- /dev/null +++ b/cmd/chunks-inspect/README.md @@ -0,0 +1,4 @@ +# Tools for inspecting Loki chunks + +This tool can parse Loki chunks and print details from them. Useful for Loki developers. + From de787d551e87cc6444a2ee3dfca9791596db868e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 28 Jan 2020 14:24:58 +0100 Subject: [PATCH 3/3] Added .gitignore. Added more description to README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- cmd/chunks-inspect/.gitignore | 1 + cmd/chunks-inspect/README.md | 65 +++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 cmd/chunks-inspect/.gitignore diff --git a/cmd/chunks-inspect/.gitignore b/cmd/chunks-inspect/.gitignore new file mode 100644 index 000000000000..3c726ec69dca --- /dev/null +++ b/cmd/chunks-inspect/.gitignore @@ -0,0 +1 @@ +chunks-inspect diff --git a/cmd/chunks-inspect/README.md b/cmd/chunks-inspect/README.md index 7dbe878b2b2a..8854527eddfa 100644 --- a/cmd/chunks-inspect/README.md +++ b/cmd/chunks-inspect/README.md @@ -2,3 +2,68 @@ This tool can parse Loki chunks and print details from them. Useful for Loki developers. +To build the tool, simply run `go build` in this directory. Running resulting program with chunks file name gives you some basic chunks information: + +```shell +$ ./chunks-inspect db61b4eca2a5ad68\:16f89ff4164\:16f8a0cfb41\:1538ace0 + +Chunks file: db61b4eca2a5ad68:16f89ff4164:16f8a0cfb41:1538ace0 +Metadata length: 485 +Data length: 264737 +UserID: 29 +From: 2020-01-09 11:10:04.644000 UTC +Through: 2020-01-09 11:25:04.193000 UTC (14m59.549s) +Labels: + __name__ = logs + app = graphite + cluster = us-central1 + color = b + container_name = graphite + filename = /var/log/pods/metrictank_graphite-1-large-multitenant-b-5f9db68b5c-jh769_ca9a10b0-0d2d-11ea-b85a-42010a80017a/graphite/0.log + hosted_metrics = 1 + instance = graphite-1-large-multitenant-b-5f9db68b5c-jh769 + job = metrictank/graphite + namespace = metrictank + org = 1 + plan = large + pod_template_hash = 5f9db68b5c + stream = stderr +Encoding: lz4 +Blocks Metadata Checksum: 3444d7a3 OK +Found 5 block(s), use -b to show block details +Minimum time (from first block): 2020-01-09 11:10:04.644490 UTC +Maximum time (from last block): 2020-01-09 11:25:04.192368 UTC +Total size of original data: 1257319 file size: 265226 ratio: 4.74 +``` + +To print more details about individual blocks inside chunks, use `-b` parameter: + +```shell script +$ ./chunks-inspect -b db61b4eca2a5ad68\:16f89ff4164\:16f8a0cfb41\:1538ace0 + +... chunk file info, see above ... + +Block 0: position: 6, original length: 273604 (stored: 56220, ratio: 4.87), minT: 2020-01-09 11:10:04.644490 UTC maxT: 2020-01-09 11:12:53.458289 UTC, checksum: 13e73d71 OK +Block 0: digest compressed: ae657fdbb2b8be55eebe86b31a21050de2b5e568444507e5958218710ddf02fd, original: 0dad619bf3049a1152cb3153d90c6db6c3f54edbf9977753dde3c4e1b09d07b4 +Block 1: position: 56230, original length: 274703 (stored: 60861, ratio: 4.51), minT: 2020-01-09 11:12:53.461855 UTC maxT: 2020-01-09 11:16:35.420787 UTC, checksum: 55269e65 OK +Block 1: digest compressed: a7999f471f68cce0458ff9790e7e7501c5bfe14cc28661d8670b9d88aeaee96f, original: a617a9e0b6c33aeaa83833470cf6164c540a7a64258e55eec6fdff483059df6f +Block 2: position: 117095, original length: 273592 (stored: 56563, ratio: 4.84), minT: 2020-01-09 11:16:35.423228 UTC maxT: 2020-01-09 11:19:28.680048 UTC, checksum: 781dba21 OK +Block 2: digest compressed: 65b59cc61c5eeea8116ce8a8c0b0d98b4d4671e8bc91656979c93717050a18fc, original: 896cc6487365ad0590097794a202aad5c89776d1c626f2cea33c652885939ac6 +Block 3: position: 173662, original length: 273745 (stored: 57486, ratio: 4.76), minT: 2020-01-09 11:19:31.062836 UTC maxT: 2020-01-09 11:23:13.562630 UTC, checksum: 2a88a52b OK +Block 3: digest compressed: 4f51a64d0397cc806a898cd6662695620083466f234d179fef5c2d02c9766191, original: 15e8a1833ccbba9aa8374029141a054127526382423d3a63f321698ff8e087b5 +Block 4: position: 231152, original length: 161675 (stored: 33440, ratio: 4.83), minT: 2020-01-09 11:23:15.416284 UTC maxT: 2020-01-09 11:25:04.192368 UTC, checksum: 6d952296 OK +Block 4: digest compressed: 8dd12235f1d619c30a9afb66823a6c827613257773669fda6fbfe014ed623cd1, original: 1f7e8ef8eb937c87ad3ed3e24c321c40d43534cc43662f83ab493fb3391548b2 +Total size of original data: 1257319 file size: 265226 ratio: 4.74 +``` + +To also print individual log lines, use `-l` parameter. Full help: + +```shell script +$ ./chunks-inspect -h +Usage of ./chunks-inspect: + -b print block details + -l print log lines + -s store blocks, using input filename, and appending block index to it +``` + +Parameter `-s` allows you to inspect individual blocks, both in compressed format (as stored in chunk file), and original raw format.