-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Add CLI to inspect RF=1 WAL segments (#13552)
- Loading branch information
Showing
5 changed files
with
1,315 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| segment-inspect |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| # Tools for inspecting Loki chunks | ||
|
|
||
| This tool can parse Loki WAL segments and print details from them. Useful for Loki developers. | ||
|
|
||
| To build the tool, simply run `go build` in this directory. Running resulting program with chunks file name gives you some basic chunks information: | ||
|
|
||
| ```shell | ||
| $ ./segment-inspect 01J2F010G82AWFGAEC3WGKC84P | ||
|
|
||
| Segment file: 01J2F010G82AWFGAEC3WGKC84P | ||
| Compressed Filesize: 2.0 MB | ||
| Series Size: 1.8 MB | ||
| Index Size: 255 kB | ||
| Stream count: 865 | ||
| Tenant count: 7 | ||
| From: 2024-07-10 18:55:47.006697 UTC | ||
| To: 2024-07-10 18:55:51.454554 UTC | ||
| Duration: 4.447857915s | ||
| ``` | ||
|
|
||
| To print the individual streams contained in this segment, use `-s` parameter: | ||
|
|
||
| ```shell script | ||
| $ ./segment-inspect -s 01J2F010G82AWFGAEC3WGKC84P | ||
|
|
||
| ... segment file info, see above ... | ||
|
|
||
| {__loki_tenant__="29", cluster="dev-us-central-0", container="loki-canary", controller_revision_hash="9cbf8c4bb", job="promtail-ops/loki-canary", name="loki-canary", namespace="promtail-ops", pod="loki-canary-z6jrt", pod_template_generation="2008", service_name="loki/loki-canary", stream="stdout"} | ||
| {__loki_tenant__="29", cluster="dev-us-central-0", container="loki-canary", controller_revision_hash="9cbf8c4bb", job="promtail-ops/loki-canary", name="loki-canary", namespace="promtail-ops", pod="loki-canary-zpx5v", pod_template_generation="2008", service_name="loki/loki-canary", stream="stdout"} | ||
| {__loki_tenant__="29", cluster="dev-us-central-0", container="metadata-forwarder", job="metadata-forwarder/metadata-forwarder", name="metadata-forwarder", namespace="metadata-forwarder", pod="metadata-forwarder-676f57978c-dpd2s", pod_template_hash="676f57978c", service_name="metadata-forwarder", stream="stderr"} | ||
| {__loki_tenant__="29", cluster="dev-us-central-0", container="metadata-forwarder", job="metadata-forwarder/metadata-forwarder", name="metadata-forwarder", namespace="metadata-forwarder", pod="metadata-forwarder-676f57978c-sbnc9", pod_template_hash="676f57978c", service_name="metadata-forwarder", stream="stderr"} | ||
| {__loki_tenant__="29", cluster="dev-us-central-0", container="mimir-read", gossip_ring_member="true", job="mimir-dev-09/mimir-read", name="mimir-read", namespace="mimir-dev-09", pod="mimir-read-74d58cd64c-4kg55", pod_template_hash="74d58cd64c", service_name="mimir/mimir-read", stream="stderr"} | ||
| ... and so on ... | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| module github.com/grafana/loki/cmd/segment-inspect | ||
|
|
||
| go 1.22 | ||
|
|
||
| toolchain go1.22.2 | ||
|
|
||
| require ( | ||
| github.com/dustin/go-humanize v1.0.1 | ||
| github.com/grafana/loki/v3 v3.0.0-20240716103001-3ac130b8a152 | ||
| ) | ||
|
|
||
| require ( | ||
| github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect | ||
| github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect | ||
| github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect | ||
| github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect | ||
| github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect | ||
| github.com/Masterminds/goutils v1.1.1 // indirect | ||
| github.com/Masterminds/semver/v3 v3.2.0 // indirect | ||
| github.com/Masterminds/sprig/v3 v3.2.3 // indirect | ||
| github.com/Workiva/go-datastructures v1.1.0 // indirect | ||
| github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect | ||
| github.com/armon/go-metrics v0.4.1 // indirect | ||
| github.com/aws/aws-sdk-go v1.50.32 // indirect | ||
| github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect | ||
| github.com/beorn7/perks v1.0.1 // indirect | ||
| github.com/buger/jsonparser v1.1.1 // indirect | ||
| github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect | ||
| github.com/cespare/xxhash v1.1.0 // indirect | ||
| github.com/cespare/xxhash/v2 v2.2.0 // indirect | ||
| github.com/coreos/go-semver v0.3.0 // indirect | ||
| github.com/coreos/go-systemd/v22 v22.5.0 // indirect | ||
| github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect | ||
| github.com/dennwc/varint v1.0.0 // indirect | ||
| github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect | ||
| github.com/edsrzf/mmap-go v1.1.0 // indirect | ||
| github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect | ||
| github.com/fatih/color v1.16.0 // indirect | ||
| github.com/felixge/httpsnoop v1.0.4 // indirect | ||
| github.com/fsnotify/fsnotify v1.7.0 // indirect | ||
| github.com/go-kit/log v0.2.1 // indirect | ||
| github.com/go-logfmt/logfmt v0.6.0 // indirect | ||
| github.com/go-logr/logr v1.4.1 // indirect | ||
| github.com/go-logr/stdr v1.2.2 // indirect | ||
| github.com/go-redis/redis/v8 v8.11.5 // indirect | ||
| github.com/gogo/googleapis v1.4.0 // indirect | ||
| github.com/gogo/protobuf v1.3.2 // indirect | ||
| github.com/gogo/status v1.1.1 // indirect | ||
| github.com/golang-jwt/jwt/v5 v5.2.1 // indirect | ||
| github.com/golang/protobuf v1.5.3 // indirect | ||
| github.com/golang/snappy v0.0.4 // indirect | ||
| github.com/google/btree v1.1.2 // indirect | ||
| github.com/google/go-cmp v0.6.0 // indirect | ||
| github.com/google/uuid v1.6.0 // indirect | ||
| github.com/gorilla/mux v1.8.0 // indirect | ||
| github.com/grafana/dskit v0.0.0-20240626184720-35810fdf1c6d // indirect | ||
| github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect | ||
| github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d // indirect | ||
| github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect | ||
| github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect | ||
| github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect | ||
| github.com/hashicorp/consul/api v1.28.2 // indirect | ||
| github.com/hashicorp/errwrap v1.1.0 // indirect | ||
| github.com/hashicorp/go-cleanhttp v0.5.2 // indirect | ||
| github.com/hashicorp/go-hclog v1.6.3 // indirect | ||
| github.com/hashicorp/go-immutable-radix v1.3.1 // indirect | ||
| github.com/hashicorp/go-msgpack v0.5.5 // indirect | ||
| github.com/hashicorp/go-multierror v1.1.1 // indirect | ||
| github.com/hashicorp/go-rootcerts v1.0.2 // indirect | ||
| github.com/hashicorp/go-sockaddr v1.0.6 // indirect | ||
| github.com/hashicorp/golang-lru v0.6.0 // indirect | ||
| github.com/hashicorp/memberlist v0.5.0 // indirect | ||
| github.com/hashicorp/serf v0.10.1 // indirect | ||
| github.com/huandu/xstrings v1.3.3 // indirect | ||
| github.com/imdario/mergo v0.3.16 // indirect | ||
| github.com/jmespath/go-jmespath v0.4.0 // indirect | ||
| github.com/jpillora/backoff v1.0.0 // indirect | ||
| github.com/json-iterator/go v1.1.12 // indirect | ||
| github.com/klauspost/compress v1.17.7 // indirect | ||
| github.com/kylelemons/godebug v1.1.0 // indirect | ||
| github.com/mattn/go-colorable v0.1.13 // indirect | ||
| github.com/mattn/go-isatty v0.0.20 // indirect | ||
| github.com/miekg/dns v1.1.58 // indirect | ||
| github.com/mitchellh/copystructure v1.0.0 // indirect | ||
| github.com/mitchellh/go-homedir v1.1.0 // indirect | ||
| github.com/mitchellh/mapstructure v1.5.0 // indirect | ||
| github.com/mitchellh/reflectwalk v1.0.1 // indirect | ||
| github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect | ||
| github.com/modern-go/reflect2 v1.0.2 // indirect | ||
| github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect | ||
| github.com/oklog/ulid v1.3.1 // indirect | ||
| github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect | ||
| github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect | ||
| github.com/opentracing/opentracing-go v1.2.0 // indirect | ||
| github.com/pierrec/lz4/v4 v4.1.18 // indirect | ||
| github.com/pires/go-proxyproto v0.7.0 // indirect | ||
| github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect | ||
| github.com/pkg/errors v0.9.1 // indirect | ||
| github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect | ||
| github.com/prometheus/client_golang v1.19.0 // indirect | ||
| github.com/prometheus/client_model v0.6.0 // indirect | ||
| github.com/prometheus/common v0.49.1-0.20240306132007-4199f18c3e92 // indirect | ||
| github.com/prometheus/common/sigv4 v0.1.0 // indirect | ||
| github.com/prometheus/exporter-toolkit v0.11.0 // indirect | ||
| github.com/prometheus/procfs v0.12.0 // indirect | ||
| github.com/prometheus/prometheus v0.51.0 // indirect | ||
| github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect | ||
| github.com/sercand/kuberesolver/v5 v5.1.1 // indirect | ||
| github.com/shopspring/decimal v1.2.0 // indirect | ||
| github.com/sony/gobreaker v0.5.0 // indirect | ||
| github.com/spf13/cast v1.3.1 // indirect | ||
| github.com/stretchr/objx v0.5.2 // indirect | ||
| github.com/stretchr/testify v1.9.0 // indirect | ||
| github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect | ||
| github.com/uber/jaeger-lib v2.4.1+incompatible // indirect | ||
| go.etcd.io/etcd/api/v3 v3.5.4 // indirect | ||
| go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect | ||
| go.etcd.io/etcd/client/v3 v3.5.4 // indirect | ||
| go.opentelemetry.io/otel v1.24.0 // indirect | ||
| go.opentelemetry.io/otel/metric v1.24.0 // indirect | ||
| go.opentelemetry.io/otel/trace v1.24.0 // indirect | ||
| go.uber.org/atomic v1.11.0 // indirect | ||
| go.uber.org/goleak v1.3.0 // indirect | ||
| go.uber.org/multierr v1.11.0 // indirect | ||
| go.uber.org/zap v1.21.0 // indirect | ||
| go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect | ||
| golang.org/x/crypto v0.24.0 // indirect | ||
| golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect | ||
| golang.org/x/mod v0.17.0 // indirect | ||
| golang.org/x/net v0.26.0 // indirect | ||
| golang.org/x/oauth2 v0.18.0 // indirect | ||
| golang.org/x/sync v0.7.0 // indirect | ||
| golang.org/x/sys v0.21.0 // indirect | ||
| golang.org/x/text v0.16.0 // indirect | ||
| golang.org/x/time v0.5.0 // indirect | ||
| golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect | ||
| google.golang.org/appengine v1.6.8 // indirect | ||
| google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 // indirect | ||
| google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect | ||
| google.golang.org/grpc v1.62.1 // indirect | ||
| google.golang.org/protobuf v1.33.0 // indirect | ||
| gopkg.in/yaml.v2 v2.4.0 // indirect | ||
| gopkg.in/yaml.v3 v3.0.1 // indirect | ||
| k8s.io/apimachinery v0.29.2 // indirect | ||
| k8s.io/client-go v0.29.2 // indirect | ||
| k8s.io/klog/v2 v2.120.1 // indirect | ||
| k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect | ||
| ) |
Oops, something went wrong.