use stringified duration for columnFamily name

Author: woodsaj

Gopkg.lock

@@ -110,7 +110,7 @@ unused-packages = true
 
 [[constraint]]
   name = "github.com/raintank/dur"
-  branch = "v2"
+  revision = "master"
 
 [[constraint]]
   name = "github.com/raintank/gziper"

Gopkg.toml

@@ -8,6 +8,7 @@ func MaxChunkSpan() uint32 {
 	return Schemas.MaxChunkSpan()
 }
 
+// TTLs returns the full set of unique TTLs (in seconds) used by the current schema config.
 func TTLs() []uint32 {
 	return Schemas.TTLs()
 }

mdata/schema.go

@@ -18,6 +18,7 @@ import (
 	"github.com/grafana/metrictank/util"
 	opentracing "github.com/opentracing/opentracing-go"
 	tags "github.com/opentracing/opentracing-go/ext"
+	"github.com/raintank/dur"
 	"github.com/raintank/schema"
 	log "github.com/sirupsen/logrus"
 )
@@ -62,7 +63,7 @@ func formatRowKey(key schema.AMKey, ts uint32) string {
 }
 
 func formatFamily(ttl uint32) string {
-	return fmt.Sprintf("ret_%d", ttlToHours(ttl))
+	return dur.FormatDuration(dur)
 }
 
 func PrepareChunkData(span uint32, data []byte) []byte {
@@ -94,14 +95,6 @@ func mutationFromWriteRequest(cwr *mdata.ChunkWriteRequest) (*bigtable.Mutation,
 	return mut, len(value)
 }
 
-func ttlToHours(ttl uint32) uint32 {
-	hours := ttl / (60 * 60)
-	if hours < 1 {
-		hours = 1
-	}
-	return hours
-}
-
 type Store struct {
 	tbl              *bigtable.Table
 	client           *bigtable.Client
@@ -144,7 +137,7 @@ func NewStore(cfg *StoreConfig, ttls []uint32) (*Store, error) {
 				Families: make(map[string]bigtable.GCPolicy),
 			}
 			for _, ttl := range ttls {
-				table.Families[formatFamily(ttl)] = bigtable.MaxAgePolicy(time.Duration(ttl/(60*60)) * time.Hour)
+				table.Families[formatFamily(ttl)] = bigtable.MaxAgePolicy(time.Duration(ttl) * time.Second)
 			}
 			err := adminClient.CreateTableFromConf(ctx, &table)
 			if err != nil {
@@ -169,13 +162,13 @@ func NewStore(cfg *StoreConfig, ttls []uint32) (*Store, error) {
 					if err != nil {
 						return nil, fmt.Errorf("btStore: failed to create cf %s/%s. %s", cfg.TableName, formatFamily(ttl), err)
 					}
-					err = adminClient.SetGCPolicy(ctx, cfg.TableName, formatFamily(ttl), bigtable.MaxAgePolicy(time.Duration(ttl/(60*60))*time.Hour))
+					err = adminClient.SetGCPolicy(ctx, cfg.TableName, formatFamily(ttl), bigtable.MaxAgePolicy(time.Duration(ttl)*time.Second))
 					if err != nil {
 						return nil, fmt.Errorf("btStore: failed to set GCPolicy of %s/%s. %s", cfg.TableName, formatFamily(ttl), err)
 					}
 				} else if policy == "" {
 					log.Infof("btStore: column family %s/%s exists but has no GCPolicy. creating it", cfg.TableName, formatFamily(ttl))
-					err = adminClient.SetGCPolicy(ctx, cfg.TableName, formatFamily(ttl), bigtable.MaxAgePolicy(time.Duration(ttl/(60*60))*time.Hour))
+					err = adminClient.SetGCPolicy(ctx, cfg.TableName, formatFamily(ttl), bigtable.MaxAgePolicy(time.Duration(ttl)*time.Second))
 					if err != nil {
 						return nil, fmt.Errorf("btStore: failed to set GCPolicy of %s/%s. %s", cfg.TableName, formatFamily(ttl), err)
 					}

store/bigtable/bigtable.go

@@ -25,8 +25,10 @@ type StoreConfig struct {
 }
 
 func (cfg *StoreConfig) Validate() error {
-	// If we dont have any write threads, then we dont WriteMaxFlushSize and WriteQueueSize
+	// If we dont have any write threads, then WriteMaxFlushSize and WriteQueueSize
 	// are not used.  If we do have write threads, then we need to make sure that
+	// the the writeMaxFlushSize is not larger then the bigtable hardcoded limit of 100k
+	// and that the writeQueue size is larger then the maxFlush.
 	if cfg.WriteConcurrency > 0 {
 		if cfg.WriteMaxFlushSize > 100000 {
 			return fmt.Errorf("write-max-flush-size must be <= 100000.")