update to work with cosmosdb

cmd/mt-split-metrics-by-ttl/main.go

@@ -32,6 +32,8 @@ var (
 	cassandraUsername = flag.String("cassandra-username", "cassandra", "username for authentication")
 	cassandraPassword = flag.String("cassandra-password", "cassandra", "password for authentication")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	// hard coded to default because those have no effect in the case of this tool anyway
 	windowFactor             = 20
 	cassandraOmitReadTimeout = 60
@@ -70,7 +72,7 @@ func main() {
 		panic(fmt.Sprintf("Error creating directory: %s", err))
 	}
 
-	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, cassandraReadConcurrency, cassandraReadConcurrency, cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, windowFactor, cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, ttls)
+	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, cassandraReadConcurrency, cassandraReadConcurrency, cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, windowFactor, cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, ttls, *cassandraDisableInitialHostLookup)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to instantiate cassandra: %s", err))
 	}

cmd/mt-store-cat/main.go

@@ -53,6 +53,8 @@ var (
 	cassandraPassword        = flag.String("cassandra-password", "cassandra", "password for authentication")
 	cassandraOmitReadTimeout = flag.Int("cassandra-omit-read-timeout", 60, "if a read is older than this, it will directly be omitted without executing")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	// our own flags
 	from         = flag.String("from", "-24h", "get data from (inclusive). only for points and points-summary format")
 	to           = flag.String("to", "now", "get data until (exclusive). only for points and points-summary format")
@@ -163,7 +165,7 @@ func main() {
 		}
 	}
 
-	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, *cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil)
+	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, *cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil, *cassandraDisableInitialHostLookup)
 	if err != nil {
 		log.Fatal(4, "failed to initialize cassandra. %s", err)
 	}

cmd/mt-update-ttl/main.go

@@ -36,6 +36,8 @@ var (
 	cassandraUsername = flag.String("cassandra-username", "cassandra", "username for authentication")
 	cassandraPassword = flag.String("cassandra-password", "cassandra", "password for authentication")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	startTs    = flag.Int("start-timestamp", 0, "timestamp at which to start, defaults to 0")
 	endTs      = flag.Int("end-timestamp", math.MaxInt32, "timestamp at which to stop, defaults to int max")
 	numThreads = flag.Int("threads", 1, "number of workers to use to process data")
@@ -100,6 +102,7 @@ func NewCassandraStore() (*gocql.Session, error) {
 	cluster.ProtoVersion = *cqlProtocolVersion
 	cluster.Keyspace = *cassandraKeyspace
 	cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: *cassandraRetries}
+	cluster.DisableInitialHostLookup = *cassandraDisableInitialHostLookup
 
 	switch *cassandraHostSelectionPolicy {
 	case "roundrobin":

cmd/mt-whisper-importer-writer/main.go

@@ -94,6 +94,8 @@ var (
 
 	cassandraSchemaFile = flag.String("cassandra-schema-file", "/etc/metrictank/schema-store-cassandra.toml", "File containing the needed schemas in case database needs initializing")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	gitHash = "(none)"
 )
 
@@ -153,7 +155,7 @@ func main() {
 		log.SetLevel(log.InfoLevel)
 	}
 
-	store, err := cassandraStore.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, 60, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil)
+	store, err := cassandraStore.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, 60, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil, *cassandraDisableInitialHostLookup)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to initialize cassandra: %q", err))
 	}

docker/docker-chaos/metrictank.ini

@@ -72,6 +72,8 @@ cassandra-auth = false
 cassandra-username = cassandra
 # password for authentication
 cassandra-password = cassandra
+# instruct the driver to not attempt to get host info from the system.peers table
+cassandra-disable-initial-host-lookup = false
 
 ## Profiling and logging ##
 
@@ -356,6 +358,8 @@ password = cassandra
 create-keyspace = false
 # File containing the needed schemas in case database needs initializing
 schema-file = /etc/metrictank/schema-idx-cassandra.toml
+# instruct the driver to not attempt to get host info from the system.peers table
+disable-initial-host-lookup = false
 
 ### in-memory only
 [memory-idx]

docker/docker-cluster/metrictank.ini

@@ -72,6 +72,8 @@ cassandra-auth = false
 cassandra-username = cassandra
 # password for authentication
 cassandra-password = cassandra
+# instruct the driver to not attempt to get host info from the system.peers table
+cassandra-disable-initial-host-lookup = false
 
 ## Profiling and logging ##
 
@@ -356,6 +358,8 @@ password = cassandra
 create-keyspace = false
 # File containing the needed schemas in case database needs initializing
 schema-file = /etc/metrictank/schema-idx-cassandra.toml
+# instruct the driver to not attempt to get host info from the system.peers table
+disable-initial-host-lookup = false
 
 ### in-memory only
 [memory-idx]

docker/docker-cosmosdb/docker-compose.yml

@@ -0,0 +1,78 @@
+version: '2'
+
+services:
+  metrictank:
+    hostname: metrictank
+    image: grafana/metrictank
+    ports:
+     - "6060:6060"
+     - "2003:2003"
+    volumes:
+      - ../../build/metrictank:/usr/bin/metrictank
+      - ../../scripts/config/metrictank-docker.ini:/etc/metrictank/metrictank.ini
+      - ./storage-schemas.conf:/etc/metrictank/storage-schemas.conf
+      - ./storage-aggregation.conf:/etc/metrictank/storage-aggregation.conf
+    environment:
+     WAIT_HOSTS: kafka:9092
+     WAIT_TIMEOUT: 60
+     MT_STATS_ADDR: graphite:2003
+     MT_CASSANDRA_ADDRS: <addr>
+     MT_CASSANDRA_TIMEOUT: 30000
+     MT_CASSANDRA_SSL: "true"
+     MT_CASSANDRA_CA_PATH: ""
+     MT_CASSANDRA_HOST_VERIFICATION: "false"
+     MT_CASSANDRA_DISABLE_INITIAL_HOST_LOOKUP: "true"
+     MT_CASSANDRA_AUTH: "true"
+     MT_CASSANDRA_USERNAME: <user>
+     MT_CASSANDRA_PASSWORD: <pass>
+     MT_CASSANDRA_IDX_HOSTS: <addr>
+     MT_CASSANDRA_IDX_TIMEOUT: "30s"
+     MT_CASSANDRA_IDX_DISABLE_INITIAL_HOST_LOOKUP: "true"
+     MT_CASSANDRA_IDX_NUM_CONNS: 1
+     MT_CASSANDRA_IDX_SSL: "true"
+     MT_CASSANDRA_IDX_CA_PATH: ""
+     MT_CASSANDRA_IDX_HOST_VERIFICATION: "false"
+     MT_CASSANDRA_IDX_AUTH: "true"
+     MT_CASSANDRA_IDX_USERNAME: <user>
+     MT_CASSANDRA_IDX_PASSWORD: <pass>
+    links:
+     - kafka
+
+  kafka:
+    hostname: kafka
+    image: raintank/kafka:latest
+    environment:
+      ADVERTISED_HOST: kafka
+      NUM_PARTITIONS: 8
+    ports:
+      - "2181:2181"
+      - "9092:9092"
+      - "9999:9999"
+    volumes:
+      - /tmp/kafka-logs
+
+  graphite:
+    hostname: graphite
+    image: graphiteapp/graphite-statsd:latest
+    ports:
+     - "8080:80"
+
+  grafana:
+    hostname: grafana
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+
+  tsdb-gw:
+    hostname: tsdb-gw
+    image: raintank/tsdb-gw:latest
+    ports:
+      - "8081:80"
+    environment:
+      GW_GRAPHITE_URL: http://metrictank:6060
+      GW_METRICS_PUBLISH: "true"
+      GW_METRICS_KAFKA_COMP: snappy
+      GW_KAFKA_TCP_ADDR: kafka:9092
+      GW_STATS_ENABLED: "true"
+      GW_STATS_PREFIX: "tsdb-gw.stats.dev.tsdbgw_tsdb-gw_1"
+      GW_STATS_ADDR: "graphite:2003"

docker/docker-cosmosdb/storage-aggregation.conf

@@ -0,0 +1,17 @@
+# This config file controls which summaries are created (using which consolidation functions) for your lower-precision archives, as defined in storage-schemas.conf
+# It is an extension of http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-aggregation-conf
+# Note:
+# * This file is optional. If it is not present, we will use avg for everything
+# * Anything not matched also uses avg for everything
+# * xFilesFactor is not honored yet.  What it is in graphite is a floating point number between 0 and 1 specifying what fraction of the previous retention level's slots must have non-null values in order to aggregate to a non-null value. The default is 0.5.
+# * aggregationMethod specifies the functions used to aggregate values for the next retention level. Legal methods are avg/average, sum, min, max, and last. The default is average.
+# Unlike Graphite, you can specify multiple, as it is often handy to have different summaries available depending on what analysis you need to do.
+# When using multiple, the first one is used for reading.  In the future, we will add capabilities to select the different archives for reading.
+# * the settings configured when metrictank starts are what is applied. So you can enable or disable archives by restarting metrictank.
+#
+# see https://github.com/grafana/metrictank/blob/master/docs/consolidation.md for related info.
+
+[default]
+pattern = .*
+xFilesFactor = 0.5
+aggregationMethod = avg,min,max

docker/docker-cosmosdb/storage-schemas.conf

@@ -0,0 +1,56 @@
+# This config file sets up your retention rules.
+# It is an extension of http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
+# Note:
+# * You can have 0 to N sections
+# * The first match wins, starting from the top. If no match found, we default to single archive of minutely points, retained for 7 days in 2h chunks
+# * The patterns are unanchored regular expressions, add '^' or '$' to match the beginning or end of a pattern.
+# * When running a cluster of metrictank instances, all instances should have the same agg-settings.
+# * Unlike whisper (graphite), the config doesn't stick: if you restart metrictank with updated settings, then those
+# will be applied. The configured rollups will be saved by primary nodes and served in responses if they are ready.
+# (note in particular that if you remove archives here, we will no longer read from them)
+# * Retentions must be specified in order of increasing interval and retention
+# * The reorderBuffer an optional buffer that temporarily keeps data points in memory as raw data and allows insertion at random order. The specified value is how many datapoints, based on the raw interval specified in the first defined retention, should be kept before they are flushed out. This is useful if the metric producers cannot guarantee that the data will arrive in order, but it is relatively memory intensive. If you are unsure whether you need this, better leave it disabled to not waste memory.
+# 
+# A given rule is made up of at least 3 lines: the name, regex pattern, retentions and optionally the reorder buffer size.
+# The retentions line can specify multiple retention definitions. You need one or more, space separated.
+#
+# There are 2 formats for a single retention definition:
+# 1) 'series-interval:count-of-datapoints'                   legacy and not easy to read
+# 2) 'series-interval:retention[:chunkspan:numchunks:ready]' more friendly format with optionally 3 extra fields
+#
+#Series intervals and retentions are specified using the following suffixes:
+#
+#s - second
+#m - minute
+#h - hour
+#d - day
+#y - year
+#
+# The final 3 fields are specific to metrictank and if unspecified, use sane defaults.
+# See https://github.com/grafana/metrictank/blob/master/docs/memory-server.md for more details
+#
+# chunkspan: duration of chunks. e.g. 10min, 30min, 1h, 90min...
+# must be valid value as described here https://github.com/grafana/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
+# Defaults to a the smallest chunkspan that can hold at least 100 points.
+#
+# numchunks: number of raw chunks to keep in in-memory ring buffer
+# See https://github.com/grafana/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
+# which may be a more effective method to cache data and alleviate workload for cassandra.
+# Defaults to 2
+#
+# ready: whether the archive is ready for querying.  This is useful if you recently introduced a new archive, but it's still being populated
+# so you rather query other archives, even if they don't have the retention to serve your queries
+# Defaults to true
+#
+# Here's an example with multiple retentions:
+# [apache_busyWorkers]
+# pattern = ^servers\.www.*\.workers\.busyWorkers$
+# retentions = 1s:1d:10min:1,1m:21d,15m:5y:2h:1:false
+#
+# This example has 3 retention definitions, the first and last override some default options (to use 10minutely and 2hourly chunks and only keep one of them in memory
+# and the last rollup is marked as not ready yet for querying.
+
+[default]
+pattern = .*
+retentions = 1s:10m:2min:2,1m:20m:5min:2
+# reorderBuffer = 20

docker/docker-dev-custom-cfg-kafka/metrictank.ini

@@ -72,6 +72,8 @@ cassandra-auth = false
 cassandra-username = cassandra
 # password for authentication
 cassandra-password = cassandra
+# instruct the driver to not attempt to get host info from the system.peers table
+cassandra-disable-initial-host-lookup = false
 
 ## Profiling and logging ##
 
@@ -356,6 +358,8 @@ password = cassandra
 create-keyspace = true
 # File containing the needed schemas in case database needs initializing
 schema-file = /etc/metrictank/schema-idx-cassandra.toml
+# instruct the driver to not attempt to get host info from the system.peers table
+disable-initial-host-lookup = false
 
 ### in-memory only
 [memory-idx]

docs/config.md

@@ -102,6 +102,8 @@ cassandra-auth = false
 cassandra-username = cassandra
 # password for authentication
 cassandra-password = cassandra
+# instruct the driver to not attempt to get host info from the system.peers table
+cassandra-disable-initial-host-lookup = false
 ```
 
 ## Profiling and logging ##
@@ -418,6 +420,8 @@ password = cassandra
 create-keyspace = true
 # File containing the needed schemas in case database needs initializing
 schema-file = /etc/metrictank/schema-idx-cassandra.toml
+# instruct the driver to not attempt to get host info from the system.peers table
+disable-initial-host-lookup = false
 ```
 
 ### in-memory only

docs/tools.md

@@ -94,6 +94,8 @@ cass config flags:
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
   -create-keyspace
     	enable the creation of the index keyspace and tables, only one node needs this (default true)
+  -disable-initial-host-lookup
+    	instruct the driver to not attempt to get host info from the system.peers table
   -enabled
     	 (default true)
   -host-verification
@@ -277,6 +279,8 @@ Flags:
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
   -cassandra-create-keyspace
     	enable the creation of the metrictank keyspace (default true)
+  -cassandra-disable-initial-host-lookup
+    	instruct the driver to not attempt to get host info from the system.peers table
   -cassandra-host-selection-policy string
     	 (default "tokenaware,hostpool-epsilon-greedy")
   -cassandra-host-verification
@@ -335,6 +339,8 @@ Flags:
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
   -cassandra-create-keyspace
     	enable the creation of the metrictank keyspace (default true)
+  -cassandra-disable-initial-host-lookup
+    	instruct the driver to not attempt to get host info from the system.peers table
   -cassandra-host-selection-policy string
     	 (default "tokenaware,hostpool-epsilon-greedy")
   -cassandra-host-verification
@@ -455,6 +461,8 @@ Flags:
     	max number of concurrent reads to cassandra. (default 20)
   -cassandra-consistency string
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
+  -cassandra-disable-initial-host-lookup
+    	instruct the driver to not attempt to get host info from the system.peers table
   -cassandra-host-selection-policy string
     	 (default "tokenaware,hostpool-epsilon-greedy")
   -cassandra-host-verification
@@ -610,6 +618,8 @@ cass config flags:
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
   -create-keyspace
     	enable the creation of the index keyspace and tables, only one node needs this (default true)
+  -disable-initial-host-lookup
+    	instruct the driver to not attempt to get host info from the system.peers table
   -enabled
     	 (default true)
   -host-verification