Schema templates + scylladb docker stack

cmd/mt-index-migrate/main.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/gocql/gocql"
 	"github.com/grafana/metrictank/cluster/partitioner"
-	"github.com/grafana/metrictank/idx/cassandra"
+	"github.com/grafana/metrictank/util"
 	"github.com/raintank/worldping-api/pkg/log"
 	"gopkg.in/raintank/schema.v1"
 )
@@ -23,6 +23,7 @@ var (
 	dstKeyspace     = flag.String("dst-keyspace", "raintank", "Cassandra keyspace in use on destination.")
 	partitionScheme = flag.String("partition-scheme", "byOrg", "method used for partitioning metrics. (byOrg|bySeries)")
 	numPartitions   = flag.Int("num-partitions", 1, "number of partitions in cluster")
+	schemaFile      = flag.String("schema-file", "/etc/metrictank/schema-idx-cassandra.toml", "File containing the needed schemas in case database needs initializing")
 
 	wg sync.WaitGroup
 )
@@ -64,7 +65,8 @@ func main() {
 	}
 
 	// ensure the dest table exists.
-	err = dstSession.Query(fmt.Sprintf(cassandra.TableSchema, *dstKeyspace)).Exec()
+	schemaTable := util.ReadEntry(*schemaFile, "schema_table").(string)
+	err = dstSession.Query(fmt.Sprintf(schemaTable, *dstKeyspace)).Exec()
 	if err != nil {
 		log.Fatal(4, "cassandra-idx failed to initialize cassandra table. %s", err)
 	}

cmd/mt-split-metrics-by-ttl/main.go

@@ -22,6 +22,7 @@ var (
 	cassandraRetries             = flag.Int("cassandra-retries", 0, "how many times to retry a query before failing it")
 	cqlProtocolVersion           = flag.Int("cql-protocol-version", 4, "cql protocol version to use")
 	cassandraCreateKeyspace      = flag.Bool("cassandra-create-keyspace", true, "enable the creation of the metrictank keyspace")
+	cassandraSchemaFile          = flag.String("cassandra-schema-file", "/etc/metrictank/schema-store-cassandra.toml", "File containing the needed schemas in case database needs initializing")
 
 	cassandraSSL              = flag.Bool("cassandra-ssl", false, "enable SSL connection to cassandra")
 	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/metrictank/ca.pem", "cassandra CA certificate path when using SSL")
@@ -69,7 +70,7 @@ func main() {
 		panic(fmt.Sprintf("Error creating directory: %s", err))
 	}
 
-	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, cassandraReadConcurrency, cassandraReadConcurrency, cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, windowFactor, cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, ttls)
+	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, cassandraReadConcurrency, cassandraReadConcurrency, cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, windowFactor, cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, ttls)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to instantiate cassandra: %s", err))
 	}

cmd/mt-store-cat/main.go

@@ -42,6 +42,7 @@ var (
 	cassandraRetries        = flag.Int("cassandra-retries", 0, "how many times to retry a query before failing it")
 	cqlProtocolVersion      = flag.Int("cql-protocol-version", 4, "cql protocol version to use")
 	cassandraCreateKeyspace = flag.Bool("cassandra-create-keyspace", true, "enable the creation of the metrictank keyspace")
+	cassandraSchemaFile     = flag.String("cassandra-schema-file", "/etc/metrictank/schema-store-cassandra.toml", "File containing the needed schemas in case database needs initializing")
 
 	cassandraSSL              = flag.Bool("cassandra-ssl", false, "enable SSL connection to cassandra")
 	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/metrictank/ca.pem", "cassandra CA certificate path when using SSL")
@@ -162,7 +163,7 @@ func main() {
 		}
 	}
 
-	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, *cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, nil)
+	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, *cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil)
 	if err != nil {
 		log.Fatal(4, "failed to initialize cassandra. %s", err)
 	}

cmd/mt-whisper-importer-writer/main.go

@@ -92,6 +92,8 @@ var (
 	cassandraUsername = globalFlags.String("cassandra-username", "cassandra", "username for authentication")
 	cassandraPassword = globalFlags.String("cassandra-password", "cassandra", "password for authentication")
 
+	cassandraSchemaFile = flag.String("cassandra-schema-file", "/etc/metrictank/schema-store-cassandra.toml", "File containing the needed schemas in case database needs initializing")
+
 	gitHash = "(none)"
 )
 
@@ -151,7 +153,7 @@ func main() {
 		log.SetLevel(log.InfoLevel)
 	}
 
-	store, err := cassandraStore.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, 60, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, nil)
+	store, err := cassandraStore.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, 60, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to initialize cassandra: %q", err))
 	}

docker/docker-chaos/metrictank.ini

@@ -58,7 +58,8 @@ cassandra-retries = 0
 cql-protocol-version = 4
 # enable the creation of the mdata keyspace and tables, only one node needs this
 cassandra-create-keyspace = false
-
+# File containing the needed schemas in case database needs initializing
+cassandra-schema-file = /etc/metrictank/schema-store-cassandra.toml
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
@@ -351,6 +352,8 @@ username = cassandra
 password = cassandra
 # enable the creation of the index keyspace and tables, only one node needs this
 create-keyspace = false
+# File containing the needed schemas in case database needs initializing
+schema-file = /etc/metrictank/schema-idx-cassandra.toml
 
 ### in-memory only
 [memory-idx]

docker/docker-cluster/metrictank.ini

@@ -58,7 +58,8 @@ cassandra-retries = 0
 cql-protocol-version = 4
 # enable the creation of the mdata keyspace and tables, only one node needs this
 cassandra-create-keyspace = false
-
+# File containing the needed schemas in case database needs initializing
+cassandra-schema-file = /etc/metrictank/schema-store-cassandra.toml
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
@@ -351,6 +352,8 @@ username = cassandra
 password = cassandra
 # enable the creation of the index keyspace and tables, only one node needs this
 create-keyspace = false
+# File containing the needed schemas in case database needs initializing
+schema-file = /etc/metrictank/schema-idx-cassandra.toml
 
 ### in-memory only
 [memory-idx]

docker/docker-dev-custom-cfg-kafka/metrictank.ini

@@ -58,7 +58,8 @@ cassandra-retries = 0
 cql-protocol-version = 4
 # enable the creation of the mdata keyspace and tables, only one node needs this
 cassandra-create-keyspace = true
-
+# File containing the needed schemas in case database needs initializing
+cassandra-schema-file = /etc/metrictank/schema-store-cassandra.toml
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
@@ -351,6 +352,8 @@ username = cassandra
 password = cassandra
 # enable the creation of the index keyspace and tables, only one node needs this
 create-keyspace = true
+# File containing the needed schemas in case database needs initializing
+schema-file = /etc/metrictank/schema-idx-cassandra.toml
 
 ### in-memory only
 [memory-idx]

docker/docker-dev-scylla/datasources/graphite

@@ -0,0 +1,7 @@
+{
+  "name":"graphite",
+  "type":"graphite",
+  "url":"http://graphite:80",
+  "access":"proxy",
+  "isDefault":false
+}

docker/docker-dev-scylla/datasources/metrictank

@@ -0,0 +1,7 @@
+{
+  "name":"metrictank",
+  "type":"graphite",
+  "url":"http://metrictank:6060",
+  "access":"proxy",
+  "isDefault":true
+}

docker/docker-dev-scylla/docker-compose.yml

@@ -0,0 +1,55 @@
+version: '2'
+
+services:
+  metrictank:
+    hostname: metrictank
+    image: grafana/metrictank
+    ports:
+     - "6060:6060"
+     - "2003:2003"
+    volumes:
+      - ../../build/metrictank:/usr/bin/metrictank
+      - ../../scripts/config/metrictank-docker.ini:/etc/metrictank/metrictank.ini
+      - ../../scripts/config/storage-schemas.conf:/etc/metrictank/storage-schemas.conf
+      - ../../scripts/config/storage-aggregation.conf:/etc/metrictank/storage-aggregation.conf
+      - ../../scripts/config/schema-store-scylladb.toml:/etc/metrictank/schema-store-cassandra.toml
+      - ../../scripts/config/schema-idx-scylladb.toml:/etc/metrictank/schema-idx-cassandra.toml
+
+    environment:
+      WAIT_HOSTS: cassandra:9042
+      WAIT_TIMEOUT: 60
+      MT_HTTP_MULTI_TENANT: "false"
+      MT_MEMORY_IDX_TAG_SUPPORT: "true"
+    links:
+     - cassandra
+
+  cassandra:
+    hostname: cassandra
+    image: scylladb/scylla
+    ports:
+     - "9042:9042"
+
+  graphite:
+    hostname: graphite
+    image: raintank/graphite-mt
+    ports:
+     - "8080:80"
+    environment:
+      GRAPHITE_CLUSTER_SERVERS: metrictank:6060
+      GRAPHITE_STATSD_HOST: statsdaemon
+      WSGI_PROCESSES: 4
+      WSGI_THREADS: 25
+
+  grafana:
+    hostname: grafana
+    image: grafana/grafana
+    ports:
+      - "3000:3000"
+
+  statsdaemon:
+    hostname: statsdaemon
+    image: raintank/statsdaemon
+    ports:
+     - "8125:8125/udp"
+    volumes:
+     - "../statsdaemon.ini:/etc/statsdaemon.ini"

docker/docker-dev/docker-compose.yml

@@ -12,6 +12,9 @@ services:
       - ../../scripts/config/metrictank-docker.ini:/etc/metrictank/metrictank.ini
       - ../../scripts/config/storage-schemas.conf:/etc/metrictank/storage-schemas.conf
       - ../../scripts/config/storage-aggregation.conf:/etc/metrictank/storage-aggregation.conf
+      - ../../scripts/config/schema-store-cassandra.toml:/etc/metrictank/schema-store-cassandra.toml
+      - ../../scripts/config/schema-idx-cassandra.toml:/etc/metrictank/schema-idx-cassandra.toml
+
     environment:
       WAIT_HOSTS: cassandra:9042
       WAIT_TIMEOUT: 60
@@ -52,4 +55,4 @@ services:
     ports:
      - "8125:8125/udp"
     volumes:
-     - "../statsdaemon.ini:/etc/statsdaemon.ini"
+     - "../statsdaemon.ini:/etc/statsdaemon.ini"

docs/config.md

@@ -88,6 +88,8 @@ cassandra-retries = 0
 cql-protocol-version = 4
 # enable the creation of the mdata keyspace and tables, only one node needs this
 cassandra-create-keyspace = true
+# File containing the needed schemas in case database needs initializing
+cassandra-schema-file = /etc/metrictank/schema-store-cassandra.toml
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
@@ -412,6 +414,8 @@ username = cassandra
 password = cassandra
 # enable the creation of the index keyspace and tables, only one node needs this
 create-keyspace = true
+# File containing the needed schemas in case database needs initializing
+schema-file = /etc/metrictank/schema-idx-cassandra.toml
 ```
 
 ### in-memory only

docs/tools.md

@@ -112,6 +112,8 @@ cass config flags:
     	cql protocol version to use (default 4)
   -prune-interval duration
     	Interval at which the index should be checked for stale series. (default 3h0m0s)
+  -schema-file string
+    	File containing the needed schemas in case database needs initializing (default "/etc/metrictank/schema-idx-cassandra.toml")
   -ssl
     	enable SSL connection to cassandra
   -timeout duration
@@ -164,6 +166,8 @@ Flags:
     	number of partitions in cluster (default 1)
   -partition-scheme string
     	method used for partitioning metrics. (byOrg|bySeries) (default "byOrg")
+  -schema-file string
+    	File containing the needed schemas in case database needs initializing (default "/etc/metrictank/schema-idx-cassandra.toml")
   -src-cass-addr string
     	Address of cassandra host to migrate from. (default "localhost")
   -src-keyspace string
@@ -283,6 +287,8 @@ Flags:
     	password for authentication (default "cassandra")
   -cassandra-retries int
     	how many times to retry a query before failing it
+  -cassandra-schema-file string
+    	File containing the needed schemas in case database needs initializing (default "/etc/metrictank/schema-store-cassandra.toml")
   -cassandra-ssl
     	enable SSL connection to cassandra
   -cassandra-timeout int
@@ -345,6 +351,8 @@ Flags:
     	max number of outstanding reads before reads will be dropped. This is important if you run queries that result in many reads in parallel. (default 200000)
   -cassandra-retries int
     	how many times to retry a query before failing it
+  -cassandra-schema-file string
+    	File containing the needed schemas in case database needs initializing (default "/etc/metrictank/schema-store-cassandra.toml")
   -cassandra-ssl
     	enable SSL connection to cassandra
   -cassandra-timeout int
@@ -620,6 +628,8 @@ cass config flags:
     	cql protocol version to use (default 4)
   -prune-interval duration
     	Interval at which the index should be checked for stale series. (default 3h0m0s)
+  -schema-file string
+    	File containing the needed schemas in case database needs initializing (default "/etc/metrictank/schema-idx-cassandra.toml")
   -ssl
     	enable SSL connection to cassandra
   -timeout duration

idx/cassandra/cassandra.go

@@ -13,27 +13,12 @@ import (
 	"github.com/grafana/metrictank/idx"
 	"github.com/grafana/metrictank/idx/memory"
 	"github.com/grafana/metrictank/stats"
+	"github.com/grafana/metrictank/util"
 	"github.com/raintank/worldping-api/pkg/log"
 	"github.com/rakyll/globalconf"
 	"gopkg.in/raintank/schema.v1"
 )
 
-const KeyspaceSchema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
-const TableSchema = `CREATE TABLE IF NOT EXISTS %s.metric_idx (
-    id text,
-    orgid int,
-    partition int,
-    name text,
-    metric text,
-    interval int,
-    unit text,
-    mtype text,
-    tags set<text>,
-    lastupdate int,
-    PRIMARY KEY (partition, id)
-) WITH compaction = {'class': 'SizeTieredCompactionStrategy'}
-    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`
-
 var (
 	// metric idx.cassadra.query-insert.ok is how many insert queries for a metric completed successfully (triggered by an add or an update)
 	statQueryInsertOk = stats.NewCounter32("idx.cassandra.query-insert.ok")
@@ -68,6 +53,7 @@ var (
 	auth             bool
 	hostverification bool
 	createKeyspace   bool
+	schemaFile       string
 	keyspace         string
 	hosts            string
 	capath           string
@@ -101,6 +87,7 @@ func ConfigSetup() *flag.FlagSet {
 	casIdx.DurationVar(&pruneInterval, "prune-interval", time.Hour*3, "Interval at which the index should be checked for stale series.")
 	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
 	casIdx.BoolVar(&createKeyspace, "create-keyspace", true, "enable the creation of the index keyspace and tables, only one node needs this")
+	casIdx.StringVar(&schemaFile, "schema-file", "/etc/metrictank/schema-idx-cassandra.toml", "File containing the needed schemas in case database needs initializing")
 
 	casIdx.BoolVar(&ssl, "ssl", false, "enable SSL connection to cassandra")
 	casIdx.StringVar(&capath, "ca-path", "/etc/metrictank/ca.pem", "cassandra CA certficate path when using SSL")
@@ -173,13 +160,17 @@ func (c *CasIdx) InitBare() error {
 		return fmt.Errorf("failed to create cassandra session: %s", err)
 	}
 
+	// read templates
+	schemaKeyspace := util.ReadEntry(schemaFile, "schema_keyspace").(string)
+	schemaTable := util.ReadEntry(schemaFile, "schema_table").(string)
+
 	// create the keyspace or ensure it exists
 	if createKeyspace {
-		err = tmpSession.Query(fmt.Sprintf(KeyspaceSchema, keyspace)).Exec()
+		err = tmpSession.Query(fmt.Sprintf(schemaKeyspace, keyspace)).Exec()
 		if err != nil {
 			return fmt.Errorf("failed to initialize cassandra keyspace: %s", err)
 		}
-		err = tmpSession.Query(fmt.Sprintf(TableSchema, keyspace)).Exec()
+		err = tmpSession.Query(fmt.Sprintf(schemaTable, keyspace)).Exec()
 		if err != nil {
 			return fmt.Errorf("failed to initialize cassandra table: %s", err)
 		}

metrictank-sample.ini

@@ -61,7 +61,8 @@ cassandra-retries = 0
 cql-protocol-version = 4
 # enable the creation of the mdata keyspace and tables, only one node needs this
 cassandra-create-keyspace = true
-
+# File containing the needed schemas in case database needs initializing
+cassandra-schema-file = /etc/metrictank/schema-store-cassandra.toml
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
@@ -354,6 +355,8 @@ username = cassandra
 password = cassandra
 # enable the creation of the index keyspace and tables, only one node needs this
 create-keyspace = true
+# File containing the needed schemas in case database needs initializing
+schema-file = /etc/metrictank/schema-idx-cassandra.toml
 
 ### in-memory only
 [memory-idx]