From 0dc539344122bf661741de4e7a994280bedc5b6c Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Wed, 13 Jun 2018 13:58:15 -0700 Subject: [PATCH 1/2] tsm/cache: Remove unused function parameter --- cmd/influx_inspect/buildtsi/buildtsi.go | 2 +- tsdb/engine/tsm1/cache.go | 2 +- tsdb/engine/tsm1/cache_race_test.go | 8 +- tsdb/engine/tsm1/cache_test.go | 76 ++--- tsdb/engine/tsm1/compact_test.go | 428 ++++++++++++------------ tsdb/engine/tsm1/engine.go | 2 +- 6 files changed, 259 insertions(+), 259 deletions(-) diff --git a/cmd/influx_inspect/buildtsi/buildtsi.go b/cmd/influx_inspect/buildtsi/buildtsi.go index 048069f5fa2..a245e575c9c 100644 --- a/cmd/influx_inspect/buildtsi/buildtsi.go +++ b/cmd/influx_inspect/buildtsi/buildtsi.go @@ -209,7 +209,7 @@ func (cmd *Command) processShard(sfile *tsdb.SeriesFile, dbName, rpName string, // Write out wal files. cmd.Logger.Info("building cache from wal files") - cache := tsm1.NewCache(tsdb.DefaultCacheMaxMemorySize, "") + cache := tsm1.NewCache(tsdb.DefaultCacheMaxMemorySize) loader := tsm1.NewCacheLoader(walPaths) loader.WithLogger(cmd.Logger) if err := loader.Load(cache); err != nil { diff --git a/tsdb/engine/tsm1/cache.go b/tsdb/engine/tsm1/cache.go index 026cadfb7a3..4724386c9ca 100644 --- a/tsdb/engine/tsm1/cache.go +++ b/tsdb/engine/tsm1/cache.go @@ -208,7 +208,7 @@ type Cache struct { // NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory. // Only used for engine caches, never for snapshots. -func NewCache(maxSize uint64, path string) *Cache { +func NewCache(maxSize uint64) *Cache { c := &Cache{ maxSize: maxSize, store: emptyStore{}, diff --git a/tsdb/engine/tsm1/cache_race_test.go b/tsdb/engine/tsm1/cache_race_test.go index 37c5d2a1539..2b6941e83e3 100644 --- a/tsdb/engine/tsm1/cache_race_test.go +++ b/tsdb/engine/tsm1/cache_race_test.go @@ -26,7 +26,7 @@ func TestCacheCheckConcurrentReadsAreSafe(t *testing.T) { } wg := sync.WaitGroup{} - c := tsm1.NewCache(1000000, "") + c := tsm1.NewCache(1000000) ch := make(chan struct{}) for _, s := range series { @@ -71,7 +71,7 @@ func TestCacheRace(t *testing.T) { } wg := sync.WaitGroup{} - c := tsm1.NewCache(1000000, "") + c := tsm1.NewCache(1000000) ch := make(chan struct{}) for _, s := range series { @@ -136,7 +136,7 @@ func TestCacheRace2Compacters(t *testing.T) { } wg := sync.WaitGroup{} - c := tsm1.NewCache(1000000, "") + c := tsm1.NewCache(1000000) ch := make(chan struct{}) for _, s := range series { @@ -181,7 +181,7 @@ func TestCacheRace2Compacters(t *testing.T) { c.ClearSnapshot(true) mu.Lock() defer mu.Unlock() - for k, _ := range myFiles { + for k := range myFiles { if _, ok := mapFiles[k]; !ok { errC <- fmt.Errorf("something else deleted one of my files") return diff --git a/tsdb/engine/tsm1/cache_test.go b/tsdb/engine/tsm1/cache_test.go index 33bf87249a9..0f0dff9673d 100644 --- a/tsdb/engine/tsm1/cache_test.go +++ b/tsdb/engine/tsm1/cache_test.go @@ -19,7 +19,7 @@ import ( ) func TestCache_NewCache(t *testing.T) { - c := NewCache(100, "") + c := NewCache(100) if c == nil { t.Fatalf("failed to create new cache") } @@ -42,7 +42,7 @@ func TestCache_CacheWrite(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(3*valuesSize, "") + c := NewCache(3 * valuesSize) if err := c.Write([]byte("foo"), values); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -65,7 +65,7 @@ func TestCache_CacheWrite_TypeConflict(t *testing.T) { values := Values{v0, v1} valuesSize := v0.Size() + v1.Size() - c := NewCache(uint64(2*valuesSize), "") + c := NewCache(uint64(2 * valuesSize)) if err := c.Write([]byte("foo"), values[:1]); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -87,7 +87,7 @@ func TestCache_CacheWriteMulti(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(30*valuesSize, "") + c := NewCache(30 * valuesSize) if err := c.WriteMulti(map[string][]Value{"foo": values, "bar": values}); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -104,19 +104,19 @@ func TestCache_CacheWriteMulti(t *testing.T) { // Tests that the cache stats and size are correctly maintained during writes. func TestCache_WriteMulti_Stats(t *testing.T) { limit := uint64(1) - c := NewCache(limit, "") + c := NewCache(limit) ms := NewTestStore() c.store = ms // Not enough room in the cache. v := NewValue(1, 1.0) - values := map[string][]Value{"foo": []Value{v, v}} + values := map[string][]Value{"foo": {v, v}} if got, exp := c.WriteMulti(values), ErrCacheMemorySizeLimitExceeded(uint64(v.Size()*2), limit); !reflect.DeepEqual(got, exp) { t.Fatalf("got %q, expected %q", got, exp) } // Fail one of the values in the write. - c = NewCache(50, "") + c = NewCache(50) c.init() c.store = ms @@ -127,7 +127,7 @@ func TestCache_WriteMulti_Stats(t *testing.T) { return true, nil } - values = map[string][]Value{"foo": []Value{v, v}, "bar": []Value{v}} + values = map[string][]Value{"foo": {v, v}, "bar": {v}} if got, exp := c.WriteMulti(values), errors.New("write failed"); !reflect.DeepEqual(got, exp) { t.Fatalf("got %v, expected %v", got, exp) } @@ -152,7 +152,7 @@ func TestCache_CacheWriteMulti_TypeConflict(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(3*valuesSize, "") + c := NewCache(3 * valuesSize) if err := c.WriteMulti(map[string][]Value{"foo": values[:1], "bar": values[1:]}); err == nil { t.Fatalf(" expected field type conflict") @@ -174,7 +174,7 @@ func TestCache_Cache_DeleteRange(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(30*valuesSize, "") + c := NewCache(30 * valuesSize) if err := c.WriteMulti(map[string][]Value{"foo": values, "bar": values}); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -213,7 +213,7 @@ func TestCache_DeleteRange_NoValues(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(3*valuesSize, "") + c := NewCache(3 * valuesSize) if err := c.WriteMulti(map[string][]Value{"foo": values}); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -248,7 +248,7 @@ func TestCache_DeleteRange_NotSorted(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(3*valuesSize, "") + c := NewCache(3 * valuesSize) if err := c.WriteMulti(map[string][]Value{"foo": values}); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -283,7 +283,7 @@ func TestCache_Cache_Delete(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(30*valuesSize, "") + c := NewCache(30 * valuesSize) if err := c.WriteMulti(map[string][]Value{"foo": values, "bar": values}); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -316,7 +316,7 @@ func TestCache_Cache_Delete(t *testing.T) { } func TestCache_Cache_Delete_NonExistent(t *testing.T) { - c := NewCache(1024, "") + c := NewCache(1024) c.Delete([][]byte{[]byte("bar")}) @@ -337,7 +337,7 @@ func TestCache_CacheWriteMulti_Duplicates(t *testing.T) { v5 := NewValue(5, 3.0) values1 := Values{v3, v4, v5} - c := NewCache(0, "") + c := NewCache(0) if err := c.WriteMulti(map[string][]Value{"foo": values0}); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -367,7 +367,7 @@ func TestCache_CacheValues(t *testing.T) { v3 := NewValue(1, 1.0) v4 := NewValue(4, 4.0) - c := NewCache(512, "") + c := NewCache(512) if deduped := c.Values([]byte("no such key")); deduped != nil { t.Fatalf("Values returned for no such key") } @@ -395,7 +395,7 @@ func TestCache_CacheSnapshot(t *testing.T) { v6 := NewValue(7, 5.0) v7 := NewValue(2, 5.0) - c := NewCache(512, "") + c := NewCache(512) if err := c.Write([]byte("foo"), Values{v0, v1, v2, v3}); err != nil { t.Fatalf("failed to write 3 values, key foo to cache: %s", err.Error()) } @@ -472,9 +472,9 @@ func TestCache_CacheSnapshot(t *testing.T) { // Tests that Snapshot updates statistics correctly. func TestCache_Snapshot_Stats(t *testing.T) { limit := uint64(16) - c := NewCache(limit, "") + c := NewCache(limit) - values := map[string][]Value{"foo": []Value{NewValue(1, 1.0)}} + values := map[string][]Value{"foo": {NewValue(1, 1.0)}} if err := c.WriteMulti(values); err != nil { t.Fatal(err) } @@ -504,7 +504,7 @@ func TestCache_Snapshot_Stats(t *testing.T) { } func TestCache_CacheEmptySnapshot(t *testing.T) { - c := NewCache(512, "") + c := NewCache(512) // Grab snapshot, and ensure it's as expected. snapshot, err := c.Snapshot() @@ -531,7 +531,7 @@ func TestCache_CacheWriteMemoryExceeded(t *testing.T) { v0 := NewValue(1, 1.0) v1 := NewValue(2, 2.0) - c := NewCache(uint64(v1.Size()), "") + c := NewCache(uint64(v1.Size())) if err := c.Write([]byte("foo"), Values{v0}); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -577,7 +577,7 @@ func TestCache_Deduplicate_Concurrent(t *testing.T) { } wg := sync.WaitGroup{} - c := NewCache(1000000, "") + c := NewCache(1000000) wg.Add(1) go func() { @@ -611,9 +611,9 @@ func TestCacheLoader_LoadSingle(t *testing.T) { p3 := NewValue(1, true) values := map[string][]Value{ - "foo": []Value{p1}, - "bar": []Value{p2}, - "baz": []Value{p3}, + "foo": {p1}, + "bar": {p2}, + "baz": {p3}, } entry := &WriteWALEntry{ @@ -629,7 +629,7 @@ func TestCacheLoader_LoadSingle(t *testing.T) { } // Load the cache using the segment. - cache := NewCache(1024, "") + cache := NewCache(1024) loader := NewCacheLoader([]string{f.Name()}) if err := loader.Load(cache); err != nil { t.Fatalf("failed to load cache: %s", err.Error()) @@ -652,7 +652,7 @@ func TestCacheLoader_LoadSingle(t *testing.T) { } // Reload the cache using the segment. - cache = NewCache(1024, "") + cache = NewCache(1024) loader = NewCacheLoader([]string{f.Name()}) if err := loader.Load(cache); err != nil { t.Fatalf("failed to load cache: %s", err.Error()) @@ -698,13 +698,13 @@ func TestCacheLoader_LoadDouble(t *testing.T) { } values := map[string][]Value{ - "foo": []Value{p1}, - "bar": []Value{p2}, + "foo": {p1}, + "bar": {p2}, } segmentWrite(w1, values) values = map[string][]Value{ - "baz": []Value{p3}, - "qux": []Value{p4}, + "baz": {p3}, + "qux": {p4}, } segmentWrite(w2, values) @@ -714,7 +714,7 @@ func TestCacheLoader_LoadDouble(t *testing.T) { } // Load the cache using the segments. - cache := NewCache(1024, "") + cache := NewCache(1024) loader := NewCacheLoader([]string{f1.Name(), f2.Name()}) if err := loader.Load(cache); err != nil { t.Fatalf("failed to load cache: %s", err.Error()) @@ -748,7 +748,7 @@ func TestCacheLoader_LoadDeleted(t *testing.T) { p3 := NewValue(3, 3.0) values := map[string][]Value{ - "foo": []Value{p1, p2, p3}, + "foo": {p1, p2, p3}, } entry := &WriteWALEntry{ @@ -778,7 +778,7 @@ func TestCacheLoader_LoadDeleted(t *testing.T) { } // Load the cache using the segment. - cache := NewCache(1024, "") + cache := NewCache(1024) loader := NewCacheLoader([]string{f.Name()}) if err := loader.Load(cache); err != nil { t.Fatalf("failed to load cache: %s", err.Error()) @@ -790,7 +790,7 @@ func TestCacheLoader_LoadDeleted(t *testing.T) { } // Reload the cache using the segment. - cache = NewCache(1024, "") + cache = NewCache(1024) loader = NewCacheLoader([]string{f.Name()}) if err := loader.Load(cache); err != nil { t.Fatalf("failed to load cache: %s", err.Error()) @@ -809,7 +809,7 @@ func TestCache_Split(t *testing.T) { values := Values{v0, v1, v2} valuesSize := uint64(v0.Size() + v1.Size() + v2.Size()) - c := NewCache(0, "") + c := NewCache(0) if err := c.Write([]byte("foo"), values); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -898,7 +898,7 @@ func (s *TestStore) count() int { return s.c var fvSize = uint64(NewValue(1, float64(1)).Size()) func BenchmarkCacheFloatEntries(b *testing.B) { - cache := NewCache(uint64(b.N)*fvSize, "") + cache := NewCache(uint64(b.N) * fvSize) vals := make([][]Value, b.N) for i := 0; i < b.N; i++ { vals[i] = []Value{NewValue(1, float64(i))} @@ -919,7 +919,7 @@ type points struct { func BenchmarkCacheParallelFloatEntries(b *testing.B) { c := b.N * runtime.GOMAXPROCS(0) - cache := NewCache(uint64(c)*fvSize*10, "") + cache := NewCache(uint64(c) * fvSize * 10) vals := make([]points, c) for i := 0; i < c; i++ { v := make([]Value, 10) diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go index 46483f94b19..6ffab42f355 100644 --- a/tsdb/engine/tsm1/compact_test.go +++ b/tsdb/engine/tsm1/compact_test.go @@ -23,11 +23,11 @@ func TestCompactor_Snapshot(t *testing.T) { v3 := tsm1.NewValue(2, float64(2)) points1 := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v1}, - "cpu,host=B#!~#value": []tsm1.Value{v2, v3}, + "cpu,host=A#!~#value": {v1}, + "cpu,host=B#!~#value": {v2, v3}, } - c := tsm1.NewCache(0, "") + c := tsm1.NewCache(0) for k, v := range points1 { if err := c.Write([]byte(k), v); err != nil { t.Fatalf("failed to write key foo to cache: %s", err.Error()) @@ -96,23 +96,23 @@ func TestCompactor_CompactFull(t *testing.T) { // write 3 TSM files with different data and one new point a1 := tsm1.NewValue(1, 1.1) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1}, + "cpu,host=A#!~#value": {a1}, } f1 := MustWriteTSM(dir, 1, writes) a2 := tsm1.NewValue(2, 1.2) b1 := tsm1.NewValue(1, 2.1) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a2}, - "cpu,host=B#!~#value": []tsm1.Value{b1}, + "cpu,host=A#!~#value": {a2}, + "cpu,host=B#!~#value": {b1}, } f2 := MustWriteTSM(dir, 2, writes) a3 := tsm1.NewValue(1, 1.3) c1 := tsm1.NewValue(1, 3.1) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a3}, - "cpu,host=C#!~#value": []tsm1.Value{c1}, + "cpu,host=A#!~#value": {a3}, + "cpu,host=C#!~#value": {c1}, } f3 := MustWriteTSM(dir, 3, writes) @@ -203,7 +203,7 @@ func TestCompactor_Compact_OverlappingBlocks(t *testing.T) { a3 := tsm1.NewValue(7, 1.1) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2, a3}, + "cpu,host=A#!~#value": {a1, a2, a3}, } f1 := MustWriteTSM(dir, 1, writes) @@ -212,7 +212,7 @@ func TestCompactor_Compact_OverlappingBlocks(t *testing.T) { c3 := tsm1.NewValue(9, 1.2) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{c1, c2, c3}, + "cpu,host=A#!~#value": {c1, c2, c3}, } f3 := MustWriteTSM(dir, 3, writes) @@ -274,7 +274,7 @@ func TestCompactor_Compact_OverlappingBlocksMultiple(t *testing.T) { a3 := tsm1.NewValue(7, 1.1) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2, a3}, + "cpu,host=A#!~#value": {a1, a2, a3}, } f1 := MustWriteTSM(dir, 1, writes) @@ -283,7 +283,7 @@ func TestCompactor_Compact_OverlappingBlocksMultiple(t *testing.T) { b3 := tsm1.NewValue(6, 1.2) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{b1, b2, b3}, + "cpu,host=A#!~#value": {b1, b2, b3}, } f2 := MustWriteTSM(dir, 2, writes) @@ -292,7 +292,7 @@ func TestCompactor_Compact_OverlappingBlocksMultiple(t *testing.T) { c3 := tsm1.NewValue(9, 1.2) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{c1, c2, c3}, + "cpu,host=A#!~#value": {c1, c2, c3}, } f3 := MustWriteTSM(dir, 3, writes) @@ -353,7 +353,7 @@ func TestCompactor_Compact_UnsortedBlocks(t *testing.T) { a3 := tsm1.NewValue(6, 1.1) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2, a3}, + "cpu,host=A#!~#value": {a1, a2, a3}, } f1 := MustWriteTSM(dir, 1, writes) @@ -362,7 +362,7 @@ func TestCompactor_Compact_UnsortedBlocks(t *testing.T) { b3 := tsm1.NewValue(3, 1.2) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{b1, b2, b3}, + "cpu,host=A#!~#value": {b1, b2, b3}, } f2 := MustWriteTSM(dir, 2, writes) @@ -420,7 +420,7 @@ func TestCompactor_Compact_UnsortedBlocksOverlapping(t *testing.T) { a2 := tsm1.NewValue(2, 1.1) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2}, + "cpu,host=A#!~#value": {a1, a2}, } f1 := MustWriteTSM(dir, 1, writes) @@ -428,7 +428,7 @@ func TestCompactor_Compact_UnsortedBlocksOverlapping(t *testing.T) { b2 := tsm1.NewValue(4, 1.2) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{b1, b2}, + "cpu,host=A#!~#value": {b1, b2}, } f2 := MustWriteTSM(dir, 2, writes) @@ -436,7 +436,7 @@ func TestCompactor_Compact_UnsortedBlocksOverlapping(t *testing.T) { c2 := tsm1.NewValue(2, 1.1) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{c1, c2}, + "cpu,host=A#!~#value": {c1, c2}, } f3 := MustWriteTSM(dir, 3, writes) @@ -494,19 +494,19 @@ func TestCompactor_CompactFull_SkipFullBlocks(t *testing.T) { a1 := tsm1.NewValue(1, 1.1) a2 := tsm1.NewValue(2, 1.2) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2}, + "cpu,host=A#!~#value": {a1, a2}, } f1 := MustWriteTSM(dir, 1, writes) a3 := tsm1.NewValue(3, 1.3) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a3}, + "cpu,host=A#!~#value": {a3}, } f2 := MustWriteTSM(dir, 2, writes) a4 := tsm1.NewValue(4, 1.4) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a4}, + "cpu,host=A#!~#value": {a4}, } f3 := MustWriteTSM(dir, 3, writes) @@ -589,7 +589,7 @@ func TestCompactor_CompactFull_TombstonedSkipBlock(t *testing.T) { a1 := tsm1.NewValue(1, 1.1) a2 := tsm1.NewValue(2, 1.2) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2}, + "cpu,host=A#!~#value": {a1, a2}, } f1 := MustWriteTSM(dir, 1, writes) @@ -602,13 +602,13 @@ func TestCompactor_CompactFull_TombstonedSkipBlock(t *testing.T) { a3 := tsm1.NewValue(3, 1.3) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a3}, + "cpu,host=A#!~#value": {a3}, } f2 := MustWriteTSM(dir, 2, writes) a4 := tsm1.NewValue(4, 1.4) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a4}, + "cpu,host=A#!~#value": {a4}, } f3 := MustWriteTSM(dir, 3, writes) @@ -691,7 +691,7 @@ func TestCompactor_CompactFull_TombstonedPartialBlock(t *testing.T) { a1 := tsm1.NewValue(1, 1.1) a2 := tsm1.NewValue(2, 1.2) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2}, + "cpu,host=A#!~#value": {a1, a2}, } f1 := MustWriteTSM(dir, 1, writes) @@ -705,13 +705,13 @@ func TestCompactor_CompactFull_TombstonedPartialBlock(t *testing.T) { a3 := tsm1.NewValue(3, 1.3) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a3}, + "cpu,host=A#!~#value": {a3}, } f2 := MustWriteTSM(dir, 2, writes) a4 := tsm1.NewValue(4, 1.4) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a4}, + "cpu,host=A#!~#value": {a4}, } f3 := MustWriteTSM(dir, 3, writes) @@ -798,7 +798,7 @@ func TestCompactor_CompactFull_TombstonedMultipleRanges(t *testing.T) { a4 := tsm1.NewValue(4, 1.4) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a1, a2, a3, a4}, + "cpu,host=A#!~#value": {a1, a2, a3, a4}, } f1 := MustWriteTSM(dir, 1, writes) @@ -813,13 +813,13 @@ func TestCompactor_CompactFull_TombstonedMultipleRanges(t *testing.T) { a5 := tsm1.NewValue(5, 1.5) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a5}, + "cpu,host=A#!~#value": {a5}, } f2 := MustWriteTSM(dir, 2, writes) a6 := tsm1.NewValue(6, 1.6) writes = map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{a6}, + "cpu,host=A#!~#value": {a6}, } f3 := MustWriteTSM(dir, 3, writes) @@ -983,7 +983,7 @@ func TestTSMKeyIterator_Single(t *testing.T) { v1 := tsm1.NewValue(1, 1.1) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v1}, + "cpu,host=A#!~#value": {v1}, } r := MustTSMReader(dir, 1, writes) @@ -1037,13 +1037,13 @@ func TestTSMKeyIterator_Duplicate(t *testing.T) { v2 := tsm1.NewValue(1, int64(2)) writes1 := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v1}, + "cpu,host=A#!~#value": {v1}, } r1 := MustTSMReader(dir, 1, writes1) writes2 := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v2}, + "cpu,host=A#!~#value": {v2}, } r2 := MustTSMReader(dir, 2, writes2) @@ -1090,7 +1090,7 @@ func TestTSMKeyIterator_MultipleKeysDeleted(t *testing.T) { v1 := tsm1.NewValue(2, int64(1)) points1 := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v1}, + "cpu,host=A#!~#value": {v1}, } r1 := MustTSMReader(dir, 1, points1) @@ -1102,8 +1102,8 @@ func TestTSMKeyIterator_MultipleKeysDeleted(t *testing.T) { v3 := tsm1.NewValue(1, float64(1)) points2 := map[string][]tsm1.Value{ - "cpu,host=A#!~#count": []tsm1.Value{v2}, - "cpu,host=B#!~#value": []tsm1.Value{v3}, + "cpu,host=A#!~#count": {v2}, + "cpu,host=B#!~#value": {v3}, } r2 := MustTSMReader(dir, 2, points2) @@ -1165,11 +1165,11 @@ func TestTSMKeyIterator_SingleDeletes(t *testing.T) { v6 := tsm1.NewValue(60, int64(1)) points1 := map[string][]tsm1.Value{ - "cpu,host=0#!~#value": []tsm1.Value{v1, v2}, - "cpu,host=A#!~#value": []tsm1.Value{v5, v6}, - "cpu,host=B#!~#value": []tsm1.Value{v3, v4}, - "cpu,host=C#!~#value": []tsm1.Value{v1, v2}, - "cpu,host=D#!~#value": []tsm1.Value{v1, v2}, + "cpu,host=0#!~#value": {v1, v2}, + "cpu,host=A#!~#value": {v5, v6}, + "cpu,host=B#!~#value": {v3, v4}, + "cpu,host=C#!~#value": {v1, v2}, + "cpu,host=D#!~#value": {v1, v2}, } r1 := MustTSMReader(dir, 1, points1) @@ -1241,7 +1241,7 @@ func TestTSMKeyIterator_Abort(t *testing.T) { v1 := tsm1.NewValue(1, 1.1) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v1}, + "cpu,host=A#!~#value": {v1}, } r := MustTSMReader(dir, 1, writes) @@ -1273,10 +1273,10 @@ func TestCacheKeyIterator_Single(t *testing.T) { v0 := tsm1.NewValue(1, 1.0) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v0}, + "cpu,host=A#!~#value": {v0}, } - c := tsm1.NewCache(0, "") + c := tsm1.NewCache(0) for k, v := range writes { if err := c.Write([]byte(k), v); err != nil { @@ -1321,10 +1321,10 @@ func TestCacheKeyIterator_Chunked(t *testing.T) { v1 := tsm1.NewValue(2, 2.0) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v0, v1}, + "cpu,host=A#!~#value": {v0, v1}, } - c := tsm1.NewCache(0, "") + c := tsm1.NewCache(0) for k, v := range writes { if err := c.Write([]byte(k), v); err != nil { @@ -1371,10 +1371,10 @@ func TestCacheKeyIterator_Abort(t *testing.T) { v0 := tsm1.NewValue(1, 1.0) writes := map[string][]tsm1.Value{ - "cpu,host=A#!~#value": []tsm1.Value{v0}, + "cpu,host=A#!~#value": {v0}, } - c := tsm1.NewCache(0, "") + c := tsm1.NewCache(0) for k, v := range writes { if err := c.Write([]byte(k), v); err != nil { @@ -1408,15 +1408,15 @@ func TestDefaultPlanner_Plan_Min(t *testing.T) { &fakeFileStore{ PathsFn: func() []tsm1.FileStat { return []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-1.tsm1", Size: 251 * 1024 * 1024, }, @@ -1435,31 +1435,31 @@ func TestDefaultPlanner_Plan_Min(t *testing.T) { // file that is in a larger step, the older ones will get compacted. func TestDefaultPlanner_Plan_CombineSequence(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-04.tsm1", Size: 128 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-04.tsm1", Size: 128 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-04.tsm1", Size: 128 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-04.tsm1", Size: 128 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-02.tsm1", Size: 67 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "07-02.tsm1", Size: 128 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "08-01.tsm1", Size: 251 * 1024 * 1024, }, @@ -1489,39 +1489,39 @@ func TestDefaultPlanner_Plan_CombineSequence(t *testing.T) { // Ensure that the planner grabs the smallest compaction step func TestDefaultPlanner_Plan_MultipleGroups(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-04.tsm1", Size: 64 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-04.tsm1", Size: 64 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-04.tsm1", Size: 64 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-04.tsm1", Size: 129 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-04.tsm1", Size: 129 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-04.tsm1", Size: 129 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "07-04.tsm1", Size: 129 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "08-04.tsm1", Size: 129 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "09-04.tsm1", // should be skipped Size: 129 * 1024 * 1024, }, @@ -1566,51 +1566,51 @@ func TestDefaultPlanner_Plan_MultipleGroups(t *testing.T) { // Ensure that the planner grabs the smallest compaction step func TestDefaultPlanner_PlanLevel_SmallestCompactionStep(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-03.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-03.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-03.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-03.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "07-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "08-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "09-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "10-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "11-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "12-01.tsm1", Size: 1 * 1024 * 1024, }, @@ -1639,31 +1639,31 @@ func TestDefaultPlanner_PlanLevel_SmallestCompactionStep(t *testing.T) { func TestDefaultPlanner_PlanLevel_SplitFile(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-03.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-03.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-03.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-04.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-03.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-01.tsm1", Size: 1 * 1024 * 1024, }, @@ -1692,31 +1692,31 @@ func TestDefaultPlanner_PlanLevel_SplitFile(t *testing.T) { func TestDefaultPlanner_PlanLevel_IsolatedHighLevel(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-02.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-02.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-03.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-04.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-02.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-02.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-02.tsm1", Size: 1 * 1024 * 1024, }, @@ -1739,27 +1739,27 @@ func TestDefaultPlanner_PlanLevel_IsolatedHighLevel(t *testing.T) { func TestDefaultPlanner_PlanLevel3_MinFiles(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-03.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-03.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-01.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-01.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-02.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-01.tsm1", Size: 1 * 1024 * 1024, }, @@ -1782,16 +1782,16 @@ func TestDefaultPlanner_PlanLevel3_MinFiles(t *testing.T) { func TestDefaultPlanner_PlanLevel2_MinFiles(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "02-04.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-02.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-03.tsm1", Size: 1 * 1024 * 1024, }, @@ -1814,28 +1814,28 @@ func TestDefaultPlanner_PlanLevel2_MinFiles(t *testing.T) { func TestDefaultPlanner_PlanLevel_Tombstone(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-03.tsm1", Size: 251 * 1024 * 1024, HasTombstone: true, }, - tsm1.FileStat{ + { Path: "02-03.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-01.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-01.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-02.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-01.tsm1", Size: 1 * 1024 * 1024, }, @@ -1864,35 +1864,35 @@ func TestDefaultPlanner_PlanLevel_Tombstone(t *testing.T) { func TestDefaultPlanner_PlanLevel_Multiple(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-01.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-01.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-01.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "07-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "08-01.tsm1", Size: 1 * 1024 * 1024, }, @@ -1922,67 +1922,67 @@ func TestDefaultPlanner_PlanLevel_Multiple(t *testing.T) { func TestDefaultPlanner_PlanLevel_InUse(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-01.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-01.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-01.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "07-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "08-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "09-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "10-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "11-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "12-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "13-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "14-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "15-01.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "16-01.tsm1", Size: 1 * 1024 * 1024, }, @@ -2036,15 +2036,15 @@ func TestDefaultPlanner_PlanLevel_InUse(t *testing.T) { func TestDefaultPlanner_PlanOptimize_NoLevel4(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-03.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-03.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-03.tsm1", Size: 2 * 1024 * 1024 * 1024, }, @@ -2067,31 +2067,31 @@ func TestDefaultPlanner_PlanOptimize_NoLevel4(t *testing.T) { func TestDefaultPlanner_PlanOptimize_Level4(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-04.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-04.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-04.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-04.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-03.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-04.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "07-03.tsm1", Size: 2 * 1024 * 1024 * 1024, }, @@ -2124,43 +2124,43 @@ func TestDefaultPlanner_PlanOptimize_Level4(t *testing.T) { func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-04.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-04.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-04.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-04.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-03.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-03.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "07-04.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "08-04.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "09-04.tsm1", Size: 2 * 1024 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "10-04.tsm1", Size: 2 * 1024 * 1024 * 1024, }, @@ -2205,15 +2205,15 @@ func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) { func TestDefaultPlanner_PlanOptimize_Optimized(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-03.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "01-04.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "01-05.tsm1", Size: 2 * 1024 * 1024 * 1024, }, @@ -2236,16 +2236,16 @@ func TestDefaultPlanner_PlanOptimize_Optimized(t *testing.T) { func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-04.tsm1", Size: 251 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "01-05.tsm1", Size: 1 * 1024 * 1024, HasTombstone: true, }, - tsm1.FileStat{ + { Path: "02-06.tsm1", Size: 2 * 1024 * 1024 * 1024, }, @@ -2277,27 +2277,27 @@ func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) { // have happened in some interval func TestDefaultPlanner_Plan_FullOnCold(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-01.tsm1", Size: 513 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-02.tsm1", Size: 129 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-02.tsm1", Size: 33 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-02.tsm1", Size: 1 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-02.tsm1", Size: 10 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "06-01.tsm1", Size: 2 * 1024 * 1024, }, @@ -2328,11 +2328,11 @@ func TestDefaultPlanner_Plan_FullOnCold(t *testing.T) { // allowable size func TestDefaultPlanner_Plan_SkipMaxSizeFiles(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-01.tsm1", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-02.tsm1", Size: 2049 * 1024 * 1024, }, @@ -2356,19 +2356,19 @@ func TestDefaultPlanner_Plan_SkipMaxSizeFiles(t *testing.T) { // allowable size func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) { testSet := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-05.tsm1", Size: 256 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-05.tsm1", Size: 256 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-05.tsm1", Size: 256 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-04.tsm1", Size: 256 * 1024 * 1024, }, @@ -2391,23 +2391,23 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) { // skip planning if all files are over the limit over := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-05.tsm1", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-05.tsm1", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-05.tsm1", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-05.tsm1", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-05.tsm1", Size: 2049 * 1024 * 1024, }, @@ -2448,43 +2448,43 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) { // would get repeatedly plan the same files and never stop. func TestDefaultPlanner_Plan_TwoGenLevel3(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "000002245-000001666.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002245-000001667.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002245-000001668.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002245-000001669.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002245-000001670.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002245-000001671.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002245-000001672.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002245-000001673.tsm", Size: 192631258, }, - tsm1.FileStat{ + { Path: "000002246-000000002.tsm", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "000002246-000000003.tsm", Size: 192631258, }, @@ -2509,19 +2509,19 @@ func TestDefaultPlanner_Plan_TwoGenLevel3(t *testing.T) { // size, but do not contain full blocks func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) { testSet := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-05.tsm1", Size: 256 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-05.tsm1", Size: 256 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-05.tsm1", Size: 256 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-04.tsm1", Size: 256 * 1024 * 1024, }, @@ -2548,11 +2548,11 @@ func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) { // skip planning if all files are over the limit over := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-05.tsm1", Size: 2049 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-05.tsm1", Size: 2049 * 1024 * 1024, }, @@ -2575,23 +2575,23 @@ func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) { // size even if there is a single file in the smaller step size func TestDefaultPlanner_Plan_CompactsMiddleSteps(t *testing.T) { data := []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "01-04.tsm1", Size: 64 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "02-04.tsm1", Size: 64 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "03-04.tsm1", Size: 64 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "04-04.tsm1", Size: 64 * 1024 * 1024, }, - tsm1.FileStat{ + { Path: "05-02.tsm1", Size: 2 * 1024 * 1024, }, @@ -2623,23 +2623,23 @@ func TestDefaultPlanner_Plan_LargeGeneration(t *testing.T) { &fakeFileStore{ PathsFn: func() []tsm1.FileStat { return []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "000000278-000000006.tsm", Size: 2148340232, }, - tsm1.FileStat{ + { Path: "000000278-000000007.tsm", Size: 2148356556, }, - tsm1.FileStat{ + { Path: "000000278-000000008.tsm", Size: 167780181, }, - tsm1.FileStat{ + { Path: "000000278-000047040.tsm", Size: 2148728539, }, - tsm1.FileStat{ + { Path: "000000278-000047041.tsm", Size: 701863692, }, @@ -2659,55 +2659,55 @@ func TestDefaultPlanner_Plan_ForceFull(t *testing.T) { &fakeFileStore{ PathsFn: func() []tsm1.FileStat { return []tsm1.FileStat{ - tsm1.FileStat{ + { Path: "000000001-000000001.tsm", Size: 2148340232, }, - tsm1.FileStat{ + { Path: "000000002-000000001.tsm", Size: 2148356556, }, - tsm1.FileStat{ + { Path: "000000003-000000001.tsm", Size: 167780181, }, - tsm1.FileStat{ + { Path: "000000004-000000001.tsm", Size: 2148728539, }, - tsm1.FileStat{ + { Path: "000000005-000000001.tsm", Size: 2148340232, }, - tsm1.FileStat{ + { Path: "000000006-000000001.tsm", Size: 2148356556, }, - tsm1.FileStat{ + { Path: "000000007-000000001.tsm", Size: 167780181, }, - tsm1.FileStat{ + { Path: "000000008-000000001.tsm", Size: 2148728539, }, - tsm1.FileStat{ + { Path: "000000009-000000002.tsm", Size: 701863692, }, - tsm1.FileStat{ + { Path: "000000010-000000002.tsm", Size: 701863692, }, - tsm1.FileStat{ + { Path: "000000011-000000002.tsm", Size: 701863692, }, - tsm1.FileStat{ + { Path: "000000012-000000002.tsm", Size: 701863692, }, - tsm1.FileStat{ + { Path: "000000013-000000002.tsm", Size: 701863692, }, diff --git a/tsdb/engine/tsm1/engine.go b/tsdb/engine/tsm1/engine.go index 93f416c6f1a..fc378f5e504 100644 --- a/tsdb/engine/tsm1/engine.go +++ b/tsdb/engine/tsm1/engine.go @@ -209,7 +209,7 @@ func NewEngine(id uint64, idx tsdb.Index, path string, walPath string, sfile *ts if opt.FileStoreObserver != nil { fs.WithObserver(opt.FileStoreObserver) } - cache := NewCache(uint64(opt.Config.CacheMaxMemorySize), path) + cache := NewCache(uint64(opt.Config.CacheMaxMemorySize)) c := NewCompactor() c.Dir = path From 544636c815799f04379cc97fed02e3c0764e8896 Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Wed, 13 Jun 2018 15:22:11 -0700 Subject: [PATCH 2/2] TSM: Fix ShouldCompactCache without WAL --- tsdb/engine/tsm1/cache.go | 15 ++++++++++-- tsdb/engine/tsm1/engine.go | 6 ++--- tsdb/engine/tsm1/engine_test.go | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/tsdb/engine/tsm1/cache.go b/tsdb/engine/tsm1/cache.go index 4724386c9ca..af5c077501a 100644 --- a/tsdb/engine/tsm1/cache.go +++ b/tsdb/engine/tsm1/cache.go @@ -197,8 +197,9 @@ type Cache struct { // This number is the number of pending or failed WriteSnaphot attempts since the last successful one. snapshotAttempts int - stats *CacheStatistics - lastSnapshot time.Time + stats *CacheStatistics + lastSnapshot time.Time + lastWriteTime time.Time // A one time synchronization used to initial the cache with a store. Since the store can allocate a // a large amount memory across shards, we lazily create it. @@ -363,6 +364,10 @@ func (c *Cache) WriteMulti(values map[string][]Value) error { c.updateMemSize(int64(addedSize)) atomic.AddInt64(&c.stats.WriteOK, 1) + c.mu.Lock() + c.lastWriteTime = time.Now() + c.mu.Unlock() + return werr } @@ -762,6 +767,12 @@ func (cl *CacheLoader) WithLogger(log *zap.Logger) { cl.Logger = log.With(zap.String("service", "cacheloader")) } +func (c *Cache) LastWriteTime() time.Time { + c.mu.RLock() + defer c.mu.RUnlock() + return c.lastWriteTime +} + // UpdateAge updates the age statistic based on the current time. func (c *Cache) UpdateAge() { c.mu.RLock() diff --git a/tsdb/engine/tsm1/engine.go b/tsdb/engine/tsm1/engine.go index fc378f5e504..6dff5eb1740 100644 --- a/tsdb/engine/tsm1/engine.go +++ b/tsdb/engine/tsm1/engine.go @@ -1883,7 +1883,7 @@ func (e *Engine) compactCache() { case <-t.C: e.Cache.UpdateAge() - if e.ShouldCompactCache() { + if e.ShouldCompactCache(time.Now()) { start := time.Now() e.traceLogger.Info("Compacting cache", zap.String("path", e.path)) err := e.WriteSnapshot() @@ -1901,7 +1901,7 @@ func (e *Engine) compactCache() { // ShouldCompactCache returns true if the Cache is over its flush threshold // or if the passed in lastWriteTime is older than the write cold threshold. -func (e *Engine) ShouldCompactCache() bool { +func (e *Engine) ShouldCompactCache(t time.Time) bool { sz := e.Cache.Size() if sz == 0 { @@ -1912,7 +1912,7 @@ func (e *Engine) ShouldCompactCache() bool { return true } - return e.WALEnabled && time.Since(e.WAL.LastWriteTime()) > e.CacheFlushWriteColdDuration + return t.Sub(e.Cache.LastWriteTime()) > e.CacheFlushWriteColdDuration } func (e *Engine) compact(wg *sync.WaitGroup) { diff --git a/tsdb/engine/tsm1/engine_test.go b/tsdb/engine/tsm1/engine_test.go index fbb9a877a66..722f645bdfe 100644 --- a/tsdb/engine/tsm1/engine_test.go +++ b/tsdb/engine/tsm1/engine_test.go @@ -1705,6 +1705,47 @@ func TestEngine_SnapshotsDisabled(t *testing.T) { } } +func TestEngine_ShouldCompactCache(t *testing.T) { + nowTime := time.Now() + + e, err := NewEngine(inmem.IndexName) + if err != nil { + t.Fatal(err) + } + + // mock the planner so compactions don't run during the test + e.CompactionPlan = &mockPlanner{} + e.SetEnabled(false) + if err := e.Open(); err != nil { + t.Fatalf("failed to open tsm1 engine: %s", err.Error()) + } + defer e.Close() + + e.CacheFlushMemorySizeThreshold = 1024 + e.CacheFlushWriteColdDuration = time.Minute + + if e.ShouldCompactCache(nowTime) { + t.Fatal("nothing written to cache, so should not compact") + } + + if err := e.WritePointsString("m,k=v f=3i"); err != nil { + t.Fatal(err) + } + + if e.ShouldCompactCache(nowTime) { + t.Fatal("cache size < flush threshold and nothing written to FileStore, so should not compact") + } + + if !e.ShouldCompactCache(nowTime.Add(time.Hour)) { + t.Fatal("last compaction was longer than flush write cold threshold, so should compact") + } + + e.CacheFlushMemorySizeThreshold = 1 + if !e.ShouldCompactCache(nowTime) { + t.Fatal("cache size > flush threshold, so should compact") + } +} + // Ensure engine can create an ascending cursor for cache and tsm values. func TestEngine_CreateCursor_Ascending(t *testing.T) { t.Parallel()