forked from influxdata/influxdb
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cache.go
473 lines (402 loc) · 12.1 KB
/
cache.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
package tsm1
import (
"expvar"
"fmt"
"log"
"os"
"sort"
"sync"
"time"
"github.com/influxdata/influxdb"
)
var ErrCacheMemoryExceeded = fmt.Errorf("cache maximum memory size exceeded")
var ErrCacheInvalidCheckpoint = fmt.Errorf("invalid checkpoint")
// entry is a set of values and some metadata.
type entry struct {
values Values // All stored values.
needSort bool // true if the values are out of order and require deduping.
}
// newEntry returns a new instance of entry.
func newEntry() *entry {
return &entry{}
}
// add adds the given values to the entry.
func (e *entry) add(values []Value) {
// See if the new values are sorted or contain duplicate timestamps
var prevTime int64
for _, v := range values {
if v.UnixNano() <= prevTime {
e.needSort = true
break
}
prevTime = v.UnixNano()
}
// if there are existing values make sure they're all less than the first of
// the new values being added
if len(e.values) == 0 {
e.values = values
} else {
l := len(e.values)
lastValTime := e.values[l-1].UnixNano()
if lastValTime >= values[0].UnixNano() {
e.needSort = true
}
e.values = append(e.values, values...)
}
}
// deduplicate sorts and orders the entry's values. If values are already deduped and
// and sorted, the function does no work and simply returns.
func (e *entry) deduplicate() {
if !e.needSort || len(e.values) == 0 {
return
}
e.values = e.values.Deduplicate()
e.needSort = false
}
// Statistics gathered by the Cache.
const (
// levels - point in time measures
statCacheMemoryBytes = "memBytes" // level: Size of in-memory cache in bytes
statCacheDiskBytes = "diskBytes" // level: Size of on-disk snapshots in bytes
statSnapshots = "snapshotCount" // level: Number of active snapshots.
statCacheAgeMs = "cacheAgeMs" // level: Number of milliseconds since cache was last snapshoted at sample time
// counters - accumulative measures
statCachedBytes = "cachedBytes" // counter: Total number of bytes written into snapshots.
statWALCompactionTimeMs = "WALCompactionTimeMs" // counter: Total number of milliseconds spent compacting snapshots
)
// Cache maintains an in-memory store of Values for a set of keys.
type Cache struct {
mu sync.RWMutex
store map[string]*entry
size uint64
maxSize uint64
// snapshots are the cache objects that are currently being written to tsm files
// they're kept in memory while flushing so they can be queried along with the cache.
// they are read only and should never be modified
snapshot *Cache
snapshotSize uint64
statMap *expvar.Map // nil for snapshots.
lastSnapshot time.Time
}
// NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory.
// Only used for engine caches, never for snapshots
func NewCache(maxSize uint64, path string) *Cache {
c := &Cache{
maxSize: maxSize,
store: make(map[string]*entry),
statMap: influxdb.NewStatistics("tsm1_cache:"+path, "tsm1_cache", map[string]string{"path": path}),
lastSnapshot: time.Now(),
}
c.UpdateAge()
c.UpdateCompactTime(0)
c.updateCachedBytes(0)
c.updateMemSize(0)
c.updateSnapshot()
return c
}
// Write writes the set of values for the key to the cache. This function is goroutine-safe.
// It returns an error if the cache has exceeded its max size.
func (c *Cache) Write(key string, values []Value) error {
c.mu.Lock()
// Enough room in the cache?
addedSize := Values(values).Size()
newSize := c.size + uint64(addedSize)
if c.maxSize > 0 && newSize+c.snapshotSize > c.maxSize {
c.mu.Unlock()
return ErrCacheMemoryExceeded
}
c.write(key, values)
c.size = newSize
c.mu.Unlock()
// Update the memory size stat
c.updateMemSize(int64(addedSize))
return nil
}
// WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe.
// It returns an error if the cache has exceeded its max size.
func (c *Cache) WriteMulti(values map[string][]Value) error {
totalSz := 0
for _, v := range values {
totalSz += Values(v).Size()
}
// Enough room in the cache?
c.mu.RLock()
newSize := c.size + uint64(totalSz)
if c.maxSize > 0 && newSize+c.snapshotSize > c.maxSize {
c.mu.RUnlock()
return ErrCacheMemoryExceeded
}
c.mu.RUnlock()
c.mu.Lock()
for k, v := range values {
c.write(k, v)
}
c.size = newSize
c.mu.Unlock()
// Update the memory size stat
c.updateMemSize(int64(totalSz))
return nil
}
// Snapshot will take a snapshot of the current cache, add it to the slice of caches that
// are being flushed, and reset the current cache with new values
func (c *Cache) Snapshot() *Cache {
c.mu.Lock()
defer c.mu.Unlock()
// If no snapshot exists, create a new one, otherwise update the existing snapshot
if c.snapshot == nil {
c.snapshot = &Cache{
store: make(map[string]*entry),
}
}
// Append the current cache values to the snapshot
for k, e := range c.store {
if _, ok := c.snapshot.store[k]; ok {
c.snapshot.store[k].add(e.values)
} else {
c.snapshot.store[k] = e
}
c.snapshotSize += uint64(Values(e.values).Size())
}
// Reset the cache
c.store = make(map[string]*entry)
c.size = 0
c.lastSnapshot = time.Now()
c.updateMemSize(-int64(c.snapshot.Size()))
c.updateCachedBytes(c.snapshot.Size())
c.updateSnapshot()
return c.snapshot
}
// Deduplicate sorts the snapshot before returning it. The compactor and any queries
// coming in while it writes will need the values sorted
func (c *Cache) Deduplicate() {
for _, e := range c.store {
e.deduplicate()
}
}
// ClearSnapshot will remove the snapshot cache from the list of flushing caches and
// adjust the size
func (c *Cache) ClearSnapshot() {
c.mu.Lock()
defer c.mu.Unlock()
c.snapshotSize = 0
c.snapshot = nil
c.updateSnapshot()
}
// Size returns the number of point-calcuated bytes the cache currently uses.
func (c *Cache) Size() uint64 {
c.mu.RLock()
defer c.mu.RUnlock()
return c.size
}
// MaxSize returns the maximum number of bytes the cache may consume.
func (c *Cache) MaxSize() uint64 {
return c.maxSize
}
// Keys returns a sorted slice of all keys under management by the cache.
func (c *Cache) Keys() []string {
var a []string
for k, _ := range c.store {
a = append(a, k)
}
sort.Strings(a)
return a
}
// Values returns a copy of all values, deduped and sorted, for the given key.
func (c *Cache) Values(key string) Values {
c.mu.RLock()
e := c.store[key]
if e != nil && e.needSort {
// Sorting is needed, so unlock and run the merge operation with
// a write-lock. It is actually possible that the data will be
// sorted by the time the merge runs, which would mean very occasionally
// a write-lock will be held when only a read-lock is required.
c.mu.RUnlock()
return func() Values {
c.mu.Lock()
defer c.mu.Unlock()
return c.merged(key)
}()
}
// No sorting required for key, so just merge while continuing to hold read-lock.
return func() Values {
defer c.mu.RUnlock()
return c.merged(key)
}()
}
// Delete will remove the keys from the cache
func (c *Cache) Delete(keys []string) {
c.mu.Lock()
defer c.mu.Unlock()
for _, k := range keys {
delete(c.store, k)
}
}
// merged returns a copy of hot and snapshot values. The copy will be merged, deduped, and
// sorted. It assumes all necessary locks have been taken. If the caller knows that the
// the hot source data for the key will not be changed, it is safe to call this function
// with a read-lock taken. Otherwise it must be called with a write-lock taken.
func (c *Cache) merged(key string) Values {
e := c.store[key]
if e == nil {
if c.snapshot == nil {
// No values in hot cache or snapshots.
return nil
}
} else {
e.deduplicate()
}
// Build the sequence of entries that will be returned, in the correct order.
// Calculate the required size of the destination buffer.
var entries []*entry
sz := 0
if c.snapshot != nil {
snapshotEntries := c.snapshot.store[key]
if snapshotEntries != nil {
entries = append(entries, snapshotEntries)
sz += len(snapshotEntries.values)
}
}
if e != nil {
entries = append(entries, e)
sz += len(e.values)
}
// Any entries? If not, return.
if sz == 0 {
return nil
}
// Create the buffer, and copy all hot values and snapshots. Individual
// entries are sorted at this point, so now the code has to check if the
// resultant buffer will be sorted from start to finish.
var needSort bool
values := make(Values, sz)
n := 0
for _, e := range entries {
if !needSort && n > 0 {
needSort = values[n-1].UnixNano() >= e.values[0].UnixNano()
}
n += copy(values[n:], e.values)
}
if needSort {
values = values.Deduplicate()
}
return values
}
// Store returns the underlying cache store. This is not goroutine safe!
// Protect access by using the Lock and Unlock functions on Cache.
func (c *Cache) Store() map[string]*entry {
return c.store
}
func (c *Cache) Lock() {
c.mu.Lock()
}
func (c *Cache) Unlock() {
c.mu.Unlock()
}
// values returns the values for the key. It doesn't lock and assumes the data is
// already sorted. Should only be used in compact.go in the CacheKeyIterator
func (c *Cache) values(key string) Values {
e := c.store[key]
if e == nil {
return nil
}
return e.values
}
// write writes the set of values for the key to the cache. This function assumes
// the lock has been taken and does not enforce the cache size limits.
func (c *Cache) write(key string, values []Value) {
e, ok := c.store[key]
if !ok {
e = newEntry()
c.store[key] = e
}
e.add(values)
}
// CacheLoader processes a set of WAL segment files, and loads a cache with the data
// contained within those files. Processing of the supplied files take place in the
// order they exist in the files slice.
type CacheLoader struct {
files []string
Logger *log.Logger
}
// NewCacheLoader returns a new instance of a CacheLoader.
func NewCacheLoader(files []string) *CacheLoader {
return &CacheLoader{
files: files,
Logger: log.New(os.Stderr, "[cacheloader] ", log.LstdFlags),
}
}
// Load returns a cache loaded with the data contained within the segment files.
// If, during reading of a segment file, corruption is encountered, that segment
// file is truncated up to and including the last valid byte, and processing
// continues with the next segment file.
func (cl *CacheLoader) Load(cache *Cache) error {
for _, fn := range cl.files {
if err := func() error {
f, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
return err
}
// Log some information about the segments.
stat, err := os.Stat(f.Name())
if err != nil {
return err
}
cl.Logger.Printf("reading file %s, size %d", f.Name(), stat.Size())
r := NewWALSegmentReader(f)
defer r.Close()
for r.Next() {
entry, err := r.Read()
if err != nil {
n := r.Count()
cl.Logger.Printf("file %s corrupt at position %d, truncating", f.Name(), n)
if err := f.Truncate(n); err != nil {
return err
}
break
}
switch t := entry.(type) {
case *WriteWALEntry:
if err := cache.WriteMulti(t.Values); err != nil {
return err
}
case *DeleteWALEntry:
cache.Delete(t.Keys)
}
}
return nil
}(); err != nil {
return err
}
}
return nil
}
// Updates the age statistic
func (c *Cache) UpdateAge() {
c.mu.RLock()
defer c.mu.RUnlock()
ageStat := new(expvar.Int)
ageStat.Set(int64(time.Now().Sub(c.lastSnapshot) / time.Millisecond))
c.statMap.Set(statCacheAgeMs, ageStat)
}
// Updates WAL compaction time statistic
func (c *Cache) UpdateCompactTime(d time.Duration) {
c.statMap.Add(statWALCompactionTimeMs, int64(d/time.Millisecond))
}
// Update the cachedBytes counter
func (c *Cache) updateCachedBytes(b uint64) {
c.statMap.Add(statCachedBytes, int64(b))
}
// Update the memSize level
func (c *Cache) updateMemSize(b int64) {
c.statMap.Add(statCacheMemoryBytes, b)
}
// Update the snapshotsCount and the diskSize levels
func (c *Cache) updateSnapshot() {
// Update disk stats
diskSizeStat := new(expvar.Int)
diskSizeStat.Set(int64(c.snapshotSize))
c.statMap.Set(statCacheDiskBytes, diskSizeStat)
snapshotsStat := new(expvar.Int)
snapshotsStat.Set(int64(1))
c.statMap.Set(statSnapshots, snapshotsStat)
}