-
Notifications
You must be signed in to change notification settings - Fork 3.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add initial WAL implementation and tests #3569
Changes from all commits
eebdd5b
1bffb70
006403c
d4b0451
301b014
b583b89
a77a919
48ede78
6776014
9a53406
abc71ae
3348dab
a509df0
41cf76f
a3cdf0b
30bcd3e
9df3b7d
0f2d66f
e7f7be4
028d0a6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,154 @@ | ||
package main | ||
|
||
import ( | ||
"flag" | ||
"fmt" | ||
"math/rand" | ||
"net/url" | ||
"runtime" | ||
"sort" | ||
"sync" | ||
"time" | ||
|
||
"github.com/influxdb/influxdb/client" | ||
) | ||
|
||
var ( | ||
batchSize = flag.Int("batchsize", 5000, "number of points per batch") | ||
seriesCount = flag.Int("series", 100000, "number of unique series to create") | ||
pointCount = flag.Int("points", 100, "number of points per series to create") | ||
concurrency = flag.Int("concurrency", 10, "number of simultaneous writes to run") | ||
batchInterval = flag.Duration("batchinterval", 0*time.Second, "duration between batches") | ||
database = flag.String("database", "stress", "name of database") | ||
address = flag.String("addr", "localhost:8086", "IP address and port of database (e.g., localhost:8086)") | ||
) | ||
|
||
func main() { | ||
flag.Parse() | ||
runtime.GOMAXPROCS(runtime.NumCPU()) | ||
|
||
startTime := time.Now() | ||
counter := NewConcurrencyLimiter(*concurrency) | ||
|
||
u, _ := url.Parse(fmt.Sprintf("http://%s", *address)) | ||
c, err := client.NewClient(client.Config{URL: *u}) | ||
if err != nil { | ||
panic(err) | ||
} | ||
|
||
var mu sync.Mutex | ||
var wg sync.WaitGroup | ||
responseTimes := make([]int, 0) | ||
|
||
totalPoints := 0 | ||
|
||
for i := 1; i <= *pointCount; i++ { | ||
batch := &client.BatchPoints{ | ||
Database: *database, | ||
WriteConsistency: "any", | ||
Time: time.Now(), | ||
Precision: "n", | ||
} | ||
|
||
for j := 1; j <= *seriesCount; j++ { | ||
p := client.Point{ | ||
Measurement: "cpu", | ||
Tags: map[string]string{"region": "uswest", "host": fmt.Sprintf("host-%d", j)}, | ||
Fields: map[string]interface{}{"value": rand.Float64()}, | ||
} | ||
batch.Points = append(batch.Points, p) | ||
if len(batch.Points) >= *batchSize { | ||
wg.Add(1) | ||
counter.Increment() | ||
totalPoints += len(batch.Points) | ||
go func(b *client.BatchPoints, total int) { | ||
st := time.Now() | ||
if _, err := c.Write(*b); err != nil { | ||
fmt.Println("ERROR: ", err.Error()) | ||
} else { | ||
mu.Lock() | ||
responseTimes = append(responseTimes, int(time.Since(st).Nanoseconds())) | ||
mu.Unlock() | ||
} | ||
wg.Done() | ||
counter.Decrement() | ||
if total%1000000 == 0 { | ||
fmt.Printf("%d total points. %d in %s\n", total, *batchSize, time.Since(st)) | ||
} | ||
}(batch, totalPoints) | ||
|
||
batch = &client.BatchPoints{ | ||
Database: *database, | ||
WriteConsistency: "any", | ||
Precision: "n", | ||
} | ||
} | ||
} | ||
} | ||
|
||
wg.Wait() | ||
sort.Sort(sort.Reverse(sort.IntSlice(responseTimes))) | ||
|
||
total := int64(0) | ||
for _, t := range responseTimes { | ||
total += int64(t) | ||
} | ||
mean := total / int64(len(responseTimes)) | ||
|
||
fmt.Printf("Wrote %d points at average rate of %.0f\n", totalPoints, float64(totalPoints)/time.Since(startTime).Seconds()) | ||
fmt.Println("Average response time: ", time.Duration(mean)) | ||
fmt.Println("Slowest response times:") | ||
for _, r := range responseTimes[:100] { | ||
fmt.Println(time.Duration(r)) | ||
} | ||
} | ||
|
||
// ConcurrencyLimiter is a go routine safe struct that can be used to | ||
// ensure that no more than a specifid max number of goroutines are | ||
// executing. | ||
type ConcurrencyLimiter struct { | ||
inc chan chan struct{} | ||
dec chan struct{} | ||
max int | ||
count int | ||
} | ||
|
||
// NewConcurrencyLimiter returns a configured limiter that will | ||
// ensure that calls to Increment will block if the max is hit. | ||
func NewConcurrencyLimiter(max int) *ConcurrencyLimiter { | ||
c := &ConcurrencyLimiter{ | ||
inc: make(chan chan struct{}), | ||
dec: make(chan struct{}, max), | ||
max: max, | ||
} | ||
go c.handleLimits() | ||
return c | ||
} | ||
|
||
// Increment will increase the count of running goroutines by 1. | ||
// if the number is currently at the max, the call to Increment | ||
// will block until another goroutine decrements. | ||
func (c *ConcurrencyLimiter) Increment() { | ||
r := make(chan struct{}) | ||
c.inc <- r | ||
<-r | ||
} | ||
|
||
// Decrement will reduce the count of running goroutines by 1 | ||
func (c *ConcurrencyLimiter) Decrement() { | ||
c.dec <- struct{}{} | ||
} | ||
|
||
// handleLimits runs in a goroutine to manage the count of | ||
// running goroutines. | ||
func (c *ConcurrencyLimiter) handleLimits() { | ||
for { | ||
r := <-c.inc | ||
if c.count >= c.max { | ||
<-c.dec | ||
c.count-- | ||
} | ||
c.count++ | ||
r <- struct{}{} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -36,10 +36,37 @@ reporting-disabled = false | |
|
||
[data] | ||
dir = "/var/opt/influxdb/data" | ||
|
||
# The following WAL settings are for the b1 storage engine used in 0.9.2. They won't | ||
# apply to any new shards created after upgrading to a version > 0.9.3. | ||
max-wal-size = 104857600 # Maximum size the WAL can reach before a flush. Defaults to 100MB. | ||
wal-flush-interval = "10m" # Maximum time data can sit in WAL before a flush. | ||
wal-partition-flush-delay = "2s" # The delay time between each WAL partition being flushed. | ||
|
||
# These are the WAL settings for the storage engine >= 0.9.3 | ||
wal-dir = "/var/opt/influxdb/wal" | ||
wal-enable-logging = true | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just want to check that you want this logging on by default (that it is not too verbose). There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes, right now the only thing that gets logged is when a compaction is run, which is infrequent. |
||
|
||
# When a series in the WAL in-memory cache reaches this size in bytes it is marked as ready to | ||
# flush to the index | ||
# wal-ready-series-size = 25600 | ||
|
||
# Flush and compact a partition once this ratio of series are over the ready size | ||
# wal-compaction-threshold = 0.6 | ||
|
||
# Force a flush and compaction if any series in a partition gets above this size in bytes | ||
# wal-max-series-size = 2097152 | ||
|
||
# Force a flush of all series and full compaction if there have been no writes in this | ||
# amount of time. This is useful for ensuring that shards that are cold for writes don't | ||
# keep a bunch of data cached in memory and in the WAL. | ||
# wal-flush-cold-interval = "10m" | ||
|
||
# Force a partition to flush its largest series if it reaches this approximate size in | ||
# bytes. Remember there are 5 partitions so you'll need at least 5x this amount of memory. | ||
# The more memory you have, the bigger this can be. | ||
# wal-partition-size-threshold = 20971520 | ||
|
||
### | ||
### [cluster] | ||
### | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What about the
bz1
engine?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Oh, bz1 didn't exist in 0.9.2.....