Replace memberlist service in favor of cortex provided service (#2224)

* Bump cortex deps to include non-experimental memberlist service

* Replace memberlist service in favor of cortex provided service

* Fix route method param for querytee tool

* Register memberlist flags

* Add documentation and example for memberlist

cmd/querytee/main.go

@@ -63,15 +63,15 @@ func lokiReadRoutes() []querytee.Route {
 	samplesComparator.RegisterSamplesType(loghttp.ResultTypeStream, compareStreams)
 
 	return []querytee.Route{
-		{Path: "/loki/api/v1/query_range", RouteName: "api_v1_query_range", Methods: "GET", ResponseComparator: samplesComparator},
-		{Path: "/loki/api/v1/query", RouteName: "api_v1_query", Methods: "GET", ResponseComparator: samplesComparator},
-		{Path: "/loki/api/v1/label", RouteName: "api_v1_label", Methods: "GET", ResponseComparator: nil},
-		{Path: "/loki/api/v1/labels", RouteName: "api_v1_labels", Methods: "GET", ResponseComparator: nil},
-		{Path: "/loki/api/v1/label/{name}/values", RouteName: "api_v1_label_name_values", Methods: "GET", ResponseComparator: nil},
-		{Path: "/loki/api/v1/series", RouteName: "api_v1_series", Methods: "GET", ResponseComparator: nil},
-		{Path: "/api/prom/query", RouteName: "api_prom_query", Methods: "GET", ResponseComparator: samplesComparator},
-		{Path: "/api/prom/label", RouteName: "api_prom_label", Methods: "GET", ResponseComparator: nil},
-		{Path: "/api/prom/label/{name}/values", RouteName: "api_prom_label_name_values", Methods: "GET", ResponseComparator: nil},
-		{Path: "/api/prom/series", RouteName: "api_prom_series", Methods: "GET", ResponseComparator: nil},
+		{Path: "/loki/api/v1/query_range", RouteName: "api_v1_query_range", Methods: []string{"GET"}, ResponseComparator: samplesComparator},
+		{Path: "/loki/api/v1/query", RouteName: "api_v1_query", Methods: []string{"GET"}, ResponseComparator: samplesComparator},
+		{Path: "/loki/api/v1/label", RouteName: "api_v1_label", Methods: []string{"GET"}, ResponseComparator: nil},
+		{Path: "/loki/api/v1/labels", RouteName: "api_v1_labels", Methods: []string{"GET"}, ResponseComparator: nil},
+		{Path: "/loki/api/v1/label/{name}/values", RouteName: "api_v1_label_name_values", Methods: []string{"GET"}, ResponseComparator: nil},
+		{Path: "/loki/api/v1/series", RouteName: "api_v1_series", Methods: []string{"GET"}, ResponseComparator: nil},
+		{Path: "/api/prom/query", RouteName: "api_prom_query", Methods: []string{"GET"}, ResponseComparator: samplesComparator},
+		{Path: "/api/prom/label", RouteName: "api_prom_label", Methods: []string{"GET"}, ResponseComparator: nil},
+		{Path: "/api/prom/label/{name}/values", RouteName: "api_prom_label_name_values", Methods: []string{"GET"}, ResponseComparator: nil},
+		{Path: "/api/prom/series", RouteName: "api_prom_series", Methods: []string{"GET"}, ResponseComparator: nil},
 	}
 }

docs/configuration/README.md

@@ -15,6 +15,7 @@ Configuration examples can be found in the [Configuration Examples](examples.md)
 * [ingester_config](#ingester_config)
   * [lifecycler_config](#lifecycler_config)
   * [ring_config](#ring_config)
+* [memberlist_config](#memberlist_config)
 * [storage_config](#storage_config)
   * [cache_config](#cache_config)
 * [chunk_store_config](#chunk_store_config)
@@ -78,6 +79,10 @@ Supported contents and default values of `loki.yaml`:
 # key value store.
 [ingester: <ingester_config>]
 
+# Configuration for an memberlist gossip ring. Only applies if
+# store is "memberlist"
+[memberlist: <memberlist_config>]
+
 # Configures where Loki will store data.
 [storage_config: <storage_config>]
 
@@ -371,7 +376,7 @@ The `ring_config` is used to discover and connect to Ingesters.
 ```yaml
 kvstore:
   # The backend storage to use for the ring. Supported values are
-  # consul, etcd, inmemory
+  # consul, etcd, inmemory, memberlist
   store: <string>
 
   # The prefix for the keys in the store. Should end with a /.
@@ -414,6 +419,107 @@ kvstore:
 [replication_factor: <int> | default = 3]
 ```
 
+## memberlist_config
+
+The `memberlist_config` block configures the gossip ring to discover and connect
+between distributors, ingesters and queriers. The configuration is unique for all
+three components to ensure a  single shared ring.
+
+```yaml
+# Name of the node in memberlist cluster. Defaults to hostname.
+# CLI flag: -memberlist.nodename
+[node_name: <string> | default = ""]
+
+# Add random suffix to the node name.
+# CLI flag: -memberlist.randomize-node-name
+[randomize_node_name: <boolean> | default = true]
+
+# The timeout for establishing a connection with a remote node, and for
+# read/write operations. Uses memberlist LAN defaults if 0.
+# CLI flag: -memberlist.stream-timeout
+[stream_timeout: <duration> | default = 0s]
+
+# Multiplication factor used when sending out messages (factor * log(N+1)).
+# CLI flag: -memberlist.retransmit-factor
+[retransmit_factor: <int> | default = 0]
+
+# How often to use pull/push sync. Uses memberlist LAN defaults if 0.
+# CLI flag: -memberlist.pullpush-interval
+[pull_push_interval: <duration> | default = 0s]
+
+# How often to gossip. Uses memberlist LAN defaults if 0.
+# CLI flag: -memberlist.gossip-interval
+[gossip_interval: <duration> | default = 0s]
+
+# How many nodes to gossip to. Uses memberlist LAN defaults if 0.
+# CLI flag: -memberlist.gossip-nodes
+[gossip_nodes: <int> | default = 0]
+
+# How long to keep gossiping to dead nodes, to give them chance to refute their
+# death. Uses memberlist LAN defaults if 0.
+# CLI flag: -memberlist.gossip-to-dead-nodes-time
+[gossip_to_dead_nodes_time: <duration> | default = 0s]
+
+# How soon can dead node's name be reclaimed with new address. Defaults to 0,
+# which is disabled.
+# CLI flag: -memberlist.dead-node-reclaim-time
+[dead_node_reclaim_time: <duration> | default = 0s]
+
+# Other cluster members to join. Can be specified multiple times.
+# CLI flag: -memberlist.join
+[join_members: <list of string> | default = ]
+
+# Min backoff duration to join other cluster members.
+# CLI flag: -memberlist.min-join-backoff
+[min_join_backoff: <duration> | default = 1s]
+
+# Max backoff duration to join other cluster members.
+# CLI flag: -memberlist.max-join-backoff
+[max_join_backoff: <duration> | default = 1m]
+
+# Max number of retries to join other cluster members.
+# CLI flag: -memberlist.max-join-retries
+[max_join_retries: <int> | default = 10]
+
+# If this node fails to join memberlist cluster, abort.
+# CLI flag: -memberlist.abort-if-join-fails
+[abort_if_cluster_join_fails: <boolean> | default = true]
+
+# If not 0, how often to rejoin the cluster. Occasional rejoin can help to fix
+# the cluster split issue, and is harmless otherwise. For example when using
+# only few components as a seed nodes (via -memberlist.join), then it's
+# recommended to use rejoin. If -memberlist.join points to dynamic service that
+# resolves to all gossiping nodes (eg. Kubernetes headless service), then rejoin
+# is not needed.
+# CLI flag: -memberlist.rejoin-interval
+[rejoin_interval: <duration> | default = 0s]
+
+# How long to keep LEFT ingesters in the ring.
+# CLI flag: -memberlist.left-ingesters-timeout
+[left_ingesters_timeout: <duration> | default = 5m]
+
+# Timeout for leaving memberlist cluster.
+# CLI flag: -memberlist.leave-timeout
+[leave_timeout: <duration> | default = 5s]
+
+# IP address to listen on for gossip messages. Multiple addresses may be
+# specified. Defaults to 0.0.0.0
+# CLI flag: -memberlist.bind-addr
+[bind_addr: <list of string> | default = ]
+
+# Port to listen on for gossip messages.
+# CLI flag: -memberlist.bind-port
+[bind_port: <int> | default = 7946]
+
+# Timeout used when connecting to other nodes to send packet.
+# CLI flag: -memberlist.packet-dial-timeout
+[packet_dial_timeout: <duration> | default = 5s]
+
+# Timeout for writing 'packet' data.
+# CLI flag: -memberlist.packet-write-timeout
+[packet_write_timeout: <duration> | default = 5s]
+```
+
 ## storage_config
 
 The `storage_config` block configures one of many possible stores for both the

docs/configuration/examples.md

@@ -4,7 +4,8 @@
 2. [Google Cloud Storage](#google-cloud-storage)
 3. [Cassandra Index](#cassandra-index)
 4. [AWS](#aws)
-5. [Using the query-frontend](#query-frontend)
+5. [Almost zero dependencies setup with Memberlist and BoltDB Shipper](#almost-zero-dependencies-setup)
+6. [Using the query-frontend](#query-frontend)
 
 ## Complete Local config
 
@@ -146,6 +147,77 @@ storage_config:
     s3forcepathstyle: true
 ```
 
+## Almost zero dependencies setup
+
+This is a configuration to deploy Loki depending only on storage solution, e.g. an
+S3-compatible API like minio. The ring configuration is based on the gossip memberlist
+and the index is shipped to storage via [boltdb-shipper](../operations/storage/boltdb-shipper.md).
+
+
+```yaml
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+distributor:
+  ring:
+    store: memberlist
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: memberlist
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+
+memberlist:
+  abort_if_cluster_join_fails: false
+
+  # Expose this port on all distributor, ingester
+  # and querier replicas.
+  bind_port: 7946
+
+  # You can use a headless k8s service for all distributor,
+  # ingester and querier components.
+  join_members:
+  - loki-gossip-ring.loki.svc.cluster.local:7946
+
+  max_join_backoff: 1m
+  max_join_retries: 10
+  min_join_backoff: 1s
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: boltdb_shipper
+    object_store: s3
+    schema: v11
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+ boltdb_shipper:
+   active_index_directory: /data/loki/index
+   cache_location: /data/loki/index_cache
+   resync_interval: 5s
+   shared_store: s3
+
+ aws:
+   s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
+   s3forcepathstyle: true
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+```
+
 ## Query Frontend
 
 [example configuration](./query-frontend.md)

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/containerd/containerd v1.3.2 // indirect
 	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
-	github.com/cortexproject/cortex v1.1.1-0.20200609120740-6bd667db776a
+	github.com/cortexproject/cortex v1.1.1-0.20200616130854-34b45d1180c3
 	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v0.7.3-0.20190817195342-4760db040282

go.sum

@@ -233,8 +233,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cortexproject/cortex v0.6.1-0.20200228110116-92ab6cbe0995/go.mod h1:3Xa3DjJxtpXqxcMGdk850lcIRb81M0fyY1MQ6udY134=
-github.com/cortexproject/cortex v1.1.1-0.20200609120740-6bd667db776a h1:7K5ZOo5c+G+nyhG0CNeL6g5LnyjTUMykA8ceC2YgTYs=
-github.com/cortexproject/cortex v1.1.1-0.20200609120740-6bd667db776a/go.mod h1:gySKqWpKaHBAFML0dOQyglOhxlJ/Eyfslf4QNBJjPoY=
+github.com/cortexproject/cortex v1.1.1-0.20200616130854-34b45d1180c3 h1:PebLohuxr0MSgWYAc/iUCFpxIdZyISk7jKUuQKTRbRw=
+github.com/cortexproject/cortex v1.1.1-0.20200616130854-34b45d1180c3/go.mod h1:gySKqWpKaHBAFML0dOQyglOhxlJ/Eyfslf4QNBJjPoY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=

pkg/loki/loki.go

@@ -83,6 +83,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) {
 	c.Worker.RegisterFlags(f)
 	c.QueryRange.RegisterFlags(f)
 	c.RuntimeConfig.RegisterFlags(f)
+	c.MemberlistKV.RegisterFlags(f, "")
 	c.Tracing.RegisterFlags(f)
 }
 
@@ -123,7 +124,7 @@ type Loki struct {
 	frontend      *frontend.Frontend
 	stopper       queryrange.Stopper
 	runtimeConfig *runtimeconfig.Manager
-	memberlistKV  *memberlist.KVInit
+	memberlistKV  *memberlist.KVInitService
 
 	httpAuthMiddleware middleware.Interface
 }

pkg/loki/modules.go

@@ -324,11 +324,9 @@ func (t *Loki) initMemberlistKV() (services.Service, error) {
 	t.cfg.MemberlistKV.Codecs = []codec.Codec{
 		ring.GetCodec(),
 	}
-	t.memberlistKV = memberlist.NewKVInit(&t.cfg.MemberlistKV)
-	return services.NewIdleService(nil, func(_ error) error {
-		t.memberlistKV.Stop()
-		return nil
-	}), nil
+
+	t.memberlistKV = memberlist.NewKVInitService(&t.cfg.MemberlistKV)
+	return t.memberlistKV, nil
 }
 
 // activePeriodConfig type returns index type which would be applicable to logs that would be pushed starting now