Sarama update

docker/docker-dev-custom-cfg-kafka/gw/tsdb-gw.ini

@@ -6,7 +6,7 @@ api-auth-plugin = file
 kafka-tcp-addr = kafka:9092
 
 metrics-topic = mdm
-metrics-kafka-comp = none
+metrics-kafka-comp = snappy
 metrics-publish = true
 metrics-partition-scheme = bySeries
 metrics-max-in-flight = 1000000

mdata/aggmetrics.go

@@ -87,18 +87,32 @@ func (ms *AggMetrics) Get(key schema.MKey) (Metric, bool) {
 }
 
 func (ms *AggMetrics) GetOrCreate(key schema.MKey, schemaId, aggId uint16) Metric {
-	ms.Lock()
+
+	// in the most common case, it's already there and an Rlock is all we need
+	ms.RLock()
 	m, ok := ms.Metrics[key]
-	if !ok {
-		k := schema.AMKey{
-			MKey: key,
-		}
-		agg := Aggregations.Get(aggId)
-		schema := Schemas.Get(schemaId)
-		m = NewAggMetric(ms.store, ms.cachePusher, k, schema.Retentions, schema.ReorderWindow, &agg, ms.dropFirstChunk)
-		ms.Metrics[key] = m
-		metricsActive.Set(len(ms.Metrics))
+	ms.RUnlock()
+	if ok {
+		return m
+	}
+
+	// if it wasn't there, get the write lock and prepare to add it
+	// but first we need to check again if someone has added it in
+	// the meantime
+	ms.Lock()
+	m, ok = ms.Metrics[key]
+	if ok {
+		ms.Unlock()
+		return m
+	}
+	k := schema.AMKey{
+		MKey: key,
 	}
+	agg := Aggregations.Get(aggId)
+	schema := Schemas.Get(schemaId)
+	m = NewAggMetric(ms.store, ms.cachePusher, k, schema.Retentions, schema.ReorderWindow, &agg, ms.dropFirstChunk)
+	ms.Metrics[key] = m
+	metricsActive.Set(len(ms.Metrics))
 	ms.Unlock()
 	return m
 }

scripts/build.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+set -e
+
+
 # Find the directory we exist within
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 cd ${DIR}/..
@@ -25,7 +29,7 @@ OUTPUT=$BUILDDIR/metrictank
 if [ "$1" == "-race" ]
 then
   set -x
-  CGO_ENABLED=1 go build -race -ldflags "-X main.gitHash=$GITVERSION" -o $OUTPUT
+  go build -race -ldflags "-X main.gitHash=$GITVERSION" -o $OUTPUT
 else
   set -x
   go build -ldflags "-X main.gitHash=$GITVERSION" -o $OUTPUT