diff --git a/Gopkg.lock b/Gopkg.lock index 7a533399..72d17d84 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -107,12 +107,12 @@ revision = "e17e99893cb6509f428e1728281c2ad60a6b31e3" [[projects]] - branch = "master" - digest = "1:295828d2783075f3d1442ae2fccccfad2016ca711fc7afea0cd46f8e5f695fb6" - name = "github.com/codeskyblue/go-uuid" + digest = "1:0c31fa2fb2c809d61d640e28cc400087fe205df6ec9623dd1eb91a7de8d4f5d6" + name = "github.com/cespare/xxhash" packages = ["."] pruneopts = "NUT" - revision = "952abbca900b023c1a80bc522ff4795db50d9d6c" + revision = "3b82fb7d186719faeedd0c2864f868c74fbf79a1" + version = "v2.0.0" [[projects]] digest = "1:1b1179a88b2bc7b454811a1c259bf6514cfde6364f5a2d130e84e209cb405879" @@ -121,6 +121,14 @@ pruneopts = "NUT" revision = "2df174808ee097f90d259e432cc04442cf60be21" +[[projects]] + branch = "master" + digest = "1:3db74f0cf75d9759d2bdfb6cb55f7d7d372c01f847c5e1ab2fe03773dc2bf024" + name = "github.com/dgryski/go-jump" + packages = ["."] + pruneopts = "NUT" + revision = "e1f439676b570800a863c37b78d1f868f51a85fc" + [[projects]] branch = "master" digest = "1:07e80adc1161a63d37f382328ebabbf4f50e027cfaadf718515c3f39a65eb18f" @@ -218,6 +226,18 @@ pruneopts = "NUT" revision = "14cafe28513321476c73967a5a4f3454b6129c46" +[[projects]] + branch = "master" + digest = "1:8d30aa9fbf51ad8c0eb044a7b7644ac08e7dbbd039b4e40fe68978255949735a" + name = "github.com/grafana/metrictank" + packages = [ + "cluster/partitioner", + "schema", + "schema/msg", + ] + pruneopts = "NUT" + revision = "c7715cb2dd264a00e595125ae16771b5696c40d4" + [[projects]] digest = "1:ac6d01547ec4f7f673311b4663909269bfb8249952de3279799289467837c3cc" name = "github.com/jmespath/go-jmespath" @@ -307,13 +327,6 @@ pruneopts = "NUT" revision = "87a4384529e0652f5035fb5cc8095faf73ea9b0b" -[[projects]] - digest = "1:702871fd067843dac45acf878c75d14fea30ad6084c93a9babfbf79fc9369a61" - name = "github.com/raintank/metrictank" - packages = ["cluster/partitioner"] - pruneopts = "NUT" - revision = "c1b3cb1beb4aee2aa66d9d8f3f53a569be73ee73" - [[projects]] digest = "1:9229a220e3bac4ea4f4a47eea9f110b690a84ed5b3b058c00aafcb47ec8d9e03" name = "github.com/rcrowley/go-metrics" @@ -352,12 +365,12 @@ revision = "20e86122596c457c2d8a0bcd7af3158fbc7dd79e" [[projects]] - digest = "1:5f9202ad4fbceea76631df26de47b4960861ae7bc3889aece44d6aa8e5171dac" + digest = "1:c97e8a5e5c435c3287f581cdb4441ba34577ec6ead8422bfedafa6269af7abf6" name = "github.com/tinylib/msgp" packages = ["msgp"] pruneopts = "NUT" - revision = "0cea1fa86e8403be1284013014f87ab942056de8" - version = "v1.0-beta" + revision = "af6442a0fcf6e2a1b824f70dd0c734f01e817751" + version = "v1.1.0" [[projects]] branch = "master" @@ -519,17 +532,6 @@ pruneopts = "NUT" revision = "b71aced4a2a1ee80a1cbbd26b85a57623790d7f4" -[[projects]] - digest = "1:c07a5fda89d318902a5218459b17afd0034f5ba29fb2a6e376c1316de71e5a95" - name = "gopkg.in/raintank/schema.v1" - packages = [ - ".", - "msg", - ] - pruneopts = "NUT" - revision = "a323316458b5df84827551e9b6b5f61cb2de423b" - version = "v1.4" - [solve-meta] analyzer-name = "dep" analyzer-version = 1 @@ -551,17 +553,17 @@ "github.com/golang/snappy", "github.com/gorilla/handlers", "github.com/gorilla/mux", + "github.com/grafana/metrictank/cluster/partitioner", + "github.com/grafana/metrictank/schema", + "github.com/grafana/metrictank/schema/msg", "github.com/jpillora/backoff", "github.com/kisielk/og-rek", "github.com/metrics20/go-metrics20/carbon20", "github.com/prometheus/procfs", - "github.com/raintank/metrictank/cluster/partitioner", "github.com/sirupsen/logrus", "github.com/streadway/amqp", "github.com/stretchr/testify/assert", "github.com/taylorchu/toki", - "gopkg.in/raintank/schema.v1", - "gopkg.in/raintank/schema.v1/msg", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 8464b8c7..1bfe5588 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -59,8 +59,8 @@ name = "github.com/taylorchu/toki" [[constraint]] - name = "gopkg.in/raintank/schema.v1" - version = "1.4.0" + name = "github.com/grafana/metrictank" + branch = "master" [[constraint]] name = "github.com/sirupsen/logrus" diff --git a/route/grafananet.go b/route/grafananet.go index e35ae0b6..b1040b30 100644 --- a/route/grafananet.go +++ b/route/grafananet.go @@ -24,8 +24,8 @@ import ( "github.com/jpillora/backoff" log "github.com/sirupsen/logrus" - "gopkg.in/raintank/schema.v1" - "gopkg.in/raintank/schema.v1/msg" + "github.com/grafana/metrictank/schema" + "github.com/grafana/metrictank/schema/msg" ) type GrafanaNet struct { diff --git a/route/kafkamdm.go b/route/kafkamdm.go index d72650a9..53491fb8 100644 --- a/route/kafkamdm.go +++ b/route/kafkamdm.go @@ -14,22 +14,23 @@ import ( log "github.com/sirupsen/logrus" "github.com/Shopify/sarama" + "github.com/grafana/metrictank/cluster/partitioner" + "github.com/grafana/metrictank/schema" "github.com/graphite-ng/carbon-relay-ng/persister" - "github.com/raintank/metrictank/cluster/partitioner" - "gopkg.in/raintank/schema.v1" ) type KafkaMdm struct { baseRoute - saramaCfg *sarama.Config - producer sarama.SyncProducer - topic string - brokers []string - buf chan []byte - partitioner *partitioner.Kafka - schemas persister.WhisperSchemas - blocking bool - dispatch func(chan []byte, []byte, metrics.Gauge, metrics.Counter) + saramaCfg *sarama.Config + producer sarama.SyncProducer + topic string + numPartitions int32 + brokers []string + buf chan []byte + partitioner *partitioner.Kafka + schemas persister.WhisperSchemas + blocking bool + dispatch func(chan []byte, []byte, metrics.Gauge, metrics.Counter) orgId int // organisation to publish data under @@ -124,18 +125,45 @@ func NewKafkaMdm(key, prefix, sub, regex, topic, codec, schemasFile, partitionBy func (r *KafkaMdm) run() { metrics := make([]*schema.MetricData, 0, r.flushMaxNum) ticker := time.NewTicker(r.flushMaxWait) + var client sarama.Client var err error + attempts := 0 for r.producer == nil { - r.producer, err = sarama.NewSyncProducer(r.brokers, r.saramaCfg) + client, err = sarama.NewClient(r.brokers, r.saramaCfg) if err == sarama.ErrOutOfBrokers { log.Warnf("kafkaMdm %q: %s", r.key, err) // sleep before trying to connect again. time.Sleep(time.Second) + attempts++ + // fail after 300 attempts + if attempts > 300 { + log.Fatalf("kafkaMdm %q: no kafka brokers available.", r.key) + } + continue } else if err != nil { log.Fatalf("kafkaMdm %q: failed to initialize kafka producer. %s", r.key, err) } + + partitions, err := client.Partitions(r.topic) + if err != nil { + log.Fatalf("kafkaMdm %q: failed to get partitions for topic %s - %s", r.key, r.topic, err) + } + if len(partitions) < 1 { + log.Fatalf("kafkaMdm %q: retrieved 0 partitions for topic %s\nThis might indicate that kafka is not in a ready state.", r.key, r.topic) + } + + r.numPartitions = int32(len(partitions)) + + r.producer, err = sarama.NewSyncProducerFromClient(client) + if err != nil { + log.Fatalf("kafkaMdm %q: failed to initialize kafka producer. %s", r.key, err) + } } + // sarama documentation states that we need to call Close() on the client + // used to create the SyncProducer + defer client.Close() + log.Infof("kafkaMdm %q: now connected to kafka", r.key) // flushes the data to kafka and resets buffer. blocks until it succeeds @@ -154,14 +182,14 @@ func (r *KafkaMdm) run() { } size += len(data) - key, err := r.partitioner.GetPartitionKey(metric, nil) + partition, err := r.partitioner.Partition(metric, r.numPartitions) if err != nil { panic(err) } payload[i] = &sarama.ProducerMessage{ - Key: sarama.ByteEncoder(key), - Topic: r.topic, - Value: sarama.ByteEncoder(data), + Partition: partition, + Topic: r.topic, + Value: sarama.ByteEncoder(data), } } err = r.producer.SendMessages(payload) @@ -202,7 +230,7 @@ func (r *KafkaMdm) run() { r.numBuffered.Dec(1) md, err := parseMetric(buf, r.schemas, r.orgId) if err != nil { - log.Errorf("KafkaMdm %q: %s", r.key, err) + log.Errorf("KafkaMdm %q: parseMetric failed, skipping metric: %s", r.key, err) continue } md.SetId() diff --git a/route/schemas.go b/route/schemas.go index 9e77201c..35a8669f 100644 --- a/route/schemas.go +++ b/route/schemas.go @@ -7,8 +7,8 @@ import ( "strconv" "strings" + "github.com/grafana/metrictank/schema" "github.com/graphite-ng/carbon-relay-ng/persister" - "gopkg.in/raintank/schema.v1" ) func getSchemas(file string) (persister.WhisperSchemas, error) { @@ -76,7 +76,6 @@ func parseMetric(buf []byte, schemas persister.WhisperSchemas, orgId int) (*sche md := schema.MetricData{ Name: name, - Metric: name, Interval: s.Retentions[0].SecondsPerPoint(), Value: val, Unit: "unknown", @@ -85,5 +84,13 @@ func parseMetric(buf []byte, schemas persister.WhisperSchemas, orgId int) (*sche Tags: tags, OrgId: orgId, } + + // ensure MetricData is valid + // this will check for an invalid name and tags + err = md.Validate() + if err != nil { + return nil, err + } + return &md, nil } diff --git a/route/schemas_test.go b/route/schemas_test.go index 035515d4..466898fe 100644 --- a/route/schemas_test.go +++ b/route/schemas_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" + "github.com/grafana/metrictank/schema" "github.com/graphite-ng/carbon-relay-ng/persister" - "gopkg.in/raintank/schema.v1" ) func getMatchEverythingSchemas() persister.WhisperSchemas { @@ -37,7 +37,6 @@ func TestParseMetricWithTags(t *testing.T) { sort.Strings(tags) expectedMd := &schema.MetricData{ Name: name, - Metric: name, Interval: 10, Value: value, Unit: "unknown", @@ -60,7 +59,6 @@ func TestParseMetricWithoutTags(t *testing.T) { md, _ := parseMetric(line, schemas, 1) expectedMd := &schema.MetricData{ Name: name, - Metric: name, Interval: 10, Value: value, Unit: "unknown", diff --git a/vendor/github.com/cespare/xxhash/LICENSE.txt b/vendor/github.com/cespare/xxhash/LICENSE.txt new file mode 100644 index 00000000..24b53065 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/cespare/xxhash/xxhash.go b/vendor/github.com/cespare/xxhash/xxhash.go new file mode 100644 index 00000000..db0b35fb --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash.go @@ -0,0 +1,236 @@ +// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described +// at http://cyan4973.github.io/xxHash/. +package xxhash + +import ( + "encoding/binary" + "errors" + "math/bits" +) + +const ( + prime1 uint64 = 11400714785074694791 + prime2 uint64 = 14029467366897019727 + prime3 uint64 = 1609587929392839161 + prime4 uint64 = 9650029242287828579 + prime5 uint64 = 2870177450012600261 +) + +// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where +// possible in the Go code is worth a small (but measurable) performance boost +// by avoiding some MOVQs. Vars are needed for the asm and also are useful for +// convenience in the Go code in a few places where we need to intentionally +// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the +// result overflows a uint64). +var ( + prime1v = prime1 + prime2v = prime2 + prime3v = prime3 + prime4v = prime4 + prime5v = prime5 +) + +// Digest implements hash.Hash64. +type Digest struct { + v1 uint64 + v2 uint64 + v3 uint64 + v4 uint64 + total uint64 + mem [32]byte + n int // how much of mem is used +} + +// New creates a new Digest that computes the 64-bit xxHash algorithm. +func New() *Digest { + var d Digest + d.Reset() + return &d +} + +// Reset clears the Digest's state so that it can be reused. +func (d *Digest) Reset() { + d.v1 = prime1v + prime2 + d.v2 = prime2 + d.v3 = 0 + d.v4 = -prime1v + d.total = 0 + d.n = 0 +} + +// Size always returns 8 bytes. +func (d *Digest) Size() int { return 8 } + +// BlockSize always returns 32 bytes. +func (d *Digest) BlockSize() int { return 32 } + +// Write adds more data to d. It always returns len(b), nil. +func (d *Digest) Write(b []byte) (n int, err error) { + n = len(b) + d.total += uint64(n) + + if d.n+n < 32 { + // This new data doesn't even fill the current block. + copy(d.mem[d.n:], b) + d.n += n + return + } + + if d.n > 0 { + // Finish off the partial block. + copy(d.mem[d.n:], b) + d.v1 = round(d.v1, u64(d.mem[0:8])) + d.v2 = round(d.v2, u64(d.mem[8:16])) + d.v3 = round(d.v3, u64(d.mem[16:24])) + d.v4 = round(d.v4, u64(d.mem[24:32])) + b = b[32-d.n:] + d.n = 0 + } + + if len(b) >= 32 { + // One or more full blocks left. + nw := writeBlocks(d, b) + b = b[nw:] + } + + // Store any remaining partial block. + copy(d.mem[:], b) + d.n = len(b) + + return +} + +// Sum appends the current hash to b and returns the resulting slice. +func (d *Digest) Sum(b []byte) []byte { + s := d.Sum64() + return append( + b, + byte(s>>56), + byte(s>>48), + byte(s>>40), + byte(s>>32), + byte(s>>24), + byte(s>>16), + byte(s>>8), + byte(s), + ) +} + +// Sum64 returns the current hash. +func (d *Digest) Sum64() uint64 { + var h uint64 + + if d.total >= 32 { + v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4 + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = d.v3 + prime5 + } + + h += d.total + + i, end := 0, d.n + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(d.mem[i:i+8])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(d.mem[i:i+4])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for i < end { + h ^= uint64(d.mem[i]) * prime5 + h = rol11(h) * prime1 + i++ + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +const ( + magic = "xxh\x06" + marshaledSize = len(magic) + 8*5 + 32 +) + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +func (d *Digest) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize) + b = append(b, magic...) + b = appendUint64(b, d.v1) + b = appendUint64(b, d.v2) + b = appendUint64(b, d.v3) + b = appendUint64(b, d.v4) + b = appendUint64(b, d.total) + b = append(b, d.mem[:d.n]...) + b = b[:len(b)+len(d.mem)-d.n] + return b, nil +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +func (d *Digest) UnmarshalBinary(b []byte) error { + if len(b) < len(magic) || string(b[:len(magic)]) != magic { + return errors.New("xxhash: invalid hash state identifier") + } + if len(b) != marshaledSize { + return errors.New("xxhash: invalid hash state size") + } + b = b[len(magic):] + b, d.v1 = consumeUint64(b) + b, d.v2 = consumeUint64(b) + b, d.v3 = consumeUint64(b) + b, d.v4 = consumeUint64(b) + b, d.total = consumeUint64(b) + copy(d.mem[:], b) + b = b[len(d.mem):] + d.n = int(d.total % uint64(len(d.mem))) + return nil +} + +func appendUint64(b []byte, x uint64) []byte { + var a [8]byte + binary.LittleEndian.PutUint64(a[:], x) + return append(b, a[:]...) +} + +func consumeUint64(b []byte) ([]byte, uint64) { + x := u64(b) + return b[8:], x +} + +func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) } +func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } + +func round(acc, input uint64) uint64 { + acc += input * prime2 + acc = rol31(acc) + acc *= prime1 + return acc +} + +func mergeRound(acc, val uint64) uint64 { + val = round(0, val) + acc ^= val + acc = acc*prime1 + prime4 + return acc +} + +func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) } +func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) } +func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) } +func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) } +func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) } +func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) } +func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) } +func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) } diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.go b/vendor/github.com/cespare/xxhash/xxhash_amd64.go new file mode 100644 index 00000000..35318d7c --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_amd64.go @@ -0,0 +1,13 @@ +// +build !appengine +// +build gc +// +build !purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +// +//go:noescape +func Sum64(b []byte) uint64 + +//go:noescape +func writeBlocks(*Digest, []byte) int diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.s b/vendor/github.com/cespare/xxhash/xxhash_amd64.s new file mode 100644 index 00000000..d580e32a --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_amd64.s @@ -0,0 +1,215 @@ +// +build !appengine +// +build gc +// +build !purego + +#include "textflag.h" + +// Register allocation: +// AX h +// CX pointer to advance through b +// DX n +// BX loop end +// R8 v1, k1 +// R9 v2 +// R10 v3 +// R11 v4 +// R12 tmp +// R13 prime1v +// R14 prime2v +// R15 prime4v + +// round reads from and advances the buffer pointer in CX. +// It assumes that R13 has prime1v and R14 has prime2v. +#define round(r) \ + MOVQ (CX), R12 \ + ADDQ $8, CX \ + IMULQ R14, R12 \ + ADDQ R12, r \ + ROLQ $31, r \ + IMULQ R13, r + +// mergeRound applies a merge round on the two registers acc and val. +// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v. +#define mergeRound(acc, val) \ + IMULQ R14, val \ + ROLQ $31, val \ + IMULQ R13, val \ + XORQ val, acc \ + IMULQ R13, acc \ + ADDQ R15, acc + +// func Sum64(b []byte) uint64 +TEXT ·Sum64(SB), NOSPLIT, $0-32 + // Load fixed primes. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + MOVQ ·prime4v(SB), R15 + + // Load slice. + MOVQ b_base+0(FP), CX + MOVQ b_len+8(FP), DX + LEAQ (CX)(DX*1), BX + + // The first loop limit will be len(b)-32. + SUBQ $32, BX + + // Check whether we have at least one block. + CMPQ DX, $32 + JLT noBlocks + + // Set up initial state (v1, v2, v3, v4). + MOVQ R13, R8 + ADDQ R14, R8 + MOVQ R14, R9 + XORQ R10, R10 + XORQ R11, R11 + SUBQ R13, R11 + + // Loop until CX > BX. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + MOVQ R8, AX + ROLQ $1, AX + MOVQ R9, R12 + ROLQ $7, R12 + ADDQ R12, AX + MOVQ R10, R12 + ROLQ $12, R12 + ADDQ R12, AX + MOVQ R11, R12 + ROLQ $18, R12 + ADDQ R12, AX + + mergeRound(AX, R8) + mergeRound(AX, R9) + mergeRound(AX, R10) + mergeRound(AX, R11) + + JMP afterBlocks + +noBlocks: + MOVQ ·prime5v(SB), AX + +afterBlocks: + ADDQ DX, AX + + // Right now BX has len(b)-32, and we want to loop until CX > len(b)-8. + ADDQ $24, BX + + CMPQ CX, BX + JG fourByte + +wordLoop: + // Calculate k1. + MOVQ (CX), R8 + ADDQ $8, CX + IMULQ R14, R8 + ROLQ $31, R8 + IMULQ R13, R8 + + XORQ R8, AX + ROLQ $27, AX + IMULQ R13, AX + ADDQ R15, AX + + CMPQ CX, BX + JLE wordLoop + +fourByte: + ADDQ $4, BX + CMPQ CX, BX + JG singles + + MOVL (CX), R8 + ADDQ $4, CX + IMULQ R13, R8 + XORQ R8, AX + + ROLQ $23, AX + IMULQ R14, AX + ADDQ ·prime3v(SB), AX + +singles: + ADDQ $4, BX + CMPQ CX, BX + JGE finalize + +singlesLoop: + MOVBQZX (CX), R12 + ADDQ $1, CX + IMULQ ·prime5v(SB), R12 + XORQ R12, AX + + ROLQ $11, AX + IMULQ R13, AX + + CMPQ CX, BX + JL singlesLoop + +finalize: + MOVQ AX, R12 + SHRQ $33, R12 + XORQ R12, AX + IMULQ R14, AX + MOVQ AX, R12 + SHRQ $29, R12 + XORQ R12, AX + IMULQ ·prime3v(SB), AX + MOVQ AX, R12 + SHRQ $32, R12 + XORQ R12, AX + + MOVQ AX, ret+24(FP) + RET + +// writeBlocks uses the same registers as above except that it uses AX to store +// the d pointer. + +// func writeBlocks(d *Digest, b []byte) int +TEXT ·writeBlocks(SB), NOSPLIT, $0-40 + // Load fixed primes needed for round. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + + // Load slice. + MOVQ b_base+8(FP), CX + MOVQ b_len+16(FP), DX + LEAQ (CX)(DX*1), BX + SUBQ $32, BX + + // Load vN from d. + MOVQ d+0(FP), AX + MOVQ 0(AX), R8 // v1 + MOVQ 8(AX), R9 // v2 + MOVQ 16(AX), R10 // v3 + MOVQ 24(AX), R11 // v4 + + // We don't need to check the loop condition here; this function is + // always called with at least one block of data to process. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + // Copy vN back to d. + MOVQ R8, 0(AX) + MOVQ R9, 8(AX) + MOVQ R10, 16(AX) + MOVQ R11, 24(AX) + + // The number of bytes written is CX minus the old base pointer. + SUBQ b_base+8(FP), CX + MOVQ CX, ret+32(FP) + + RET diff --git a/vendor/github.com/cespare/xxhash/xxhash_other.go b/vendor/github.com/cespare/xxhash/xxhash_other.go new file mode 100644 index 00000000..4a5a8216 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_other.go @@ -0,0 +1,76 @@ +// +build !amd64 appengine !gc purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +func Sum64(b []byte) uint64 { + // A simpler version would be + // d := New() + // d.Write(b) + // return d.Sum64() + // but this is faster, particularly for small inputs. + + n := len(b) + var h uint64 + + if n >= 32 { + v1 := prime1v + prime2 + v2 := prime2 + v3 := uint64(0) + v4 := -prime1v + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = prime5 + } + + h += uint64(n) + + i, end := 0, len(b) + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(b[i:i+8:len(b)])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for ; i < end; i++ { + h ^= uint64(b[i]) * prime5 + h = rol11(h) * prime1 + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +func writeBlocks(d *Digest, b []byte) int { + v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4 + n := len(b) + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + d.v1, d.v2, d.v3, d.v4 = v1, v2, v3, v4 + return n - len(b) +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_safe.go b/vendor/github.com/cespare/xxhash/xxhash_safe.go new file mode 100644 index 00000000..fc9bea7a --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_safe.go @@ -0,0 +1,15 @@ +// +build appengine + +// This file contains the safe implementations of otherwise unsafe-using code. + +package xxhash + +// Sum64String computes the 64-bit xxHash digest of s. +func Sum64String(s string) uint64 { + return Sum64([]byte(s)) +} + +// WriteString adds more data to d. It always returns len(s), nil. +func (d *Digest) WriteString(s string) (n int, err error) { + return d.Write([]byte(s)) +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_unsafe.go b/vendor/github.com/cespare/xxhash/xxhash_unsafe.go new file mode 100644 index 00000000..53bf76ef --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_unsafe.go @@ -0,0 +1,46 @@ +// +build !appengine + +// This file encapsulates usage of unsafe. +// xxhash_safe.go contains the safe implementations. + +package xxhash + +import ( + "reflect" + "unsafe" +) + +// Notes: +// +// See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ +// for some discussion about these unsafe conversions. +// +// In the future it's possible that compiler optimizations will make these +// unsafe operations unnecessary: https://golang.org/issue/2205. +// +// Both of these wrapper functions still incur function call overhead since they +// will not be inlined. We could write Go/asm copies of Sum64 and Digest.Write +// for strings to squeeze out a bit more speed. Mid-stack inlining should +// eventually fix this. + +// Sum64String computes the 64-bit xxHash digest of s. +// It may be faster than Sum64([]byte(s)) by avoiding a copy. +func Sum64String(s string) uint64 { + var b []byte + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data + bh.Len = len(s) + bh.Cap = len(s) + return Sum64(b) +} + +// WriteString adds more data to d. It always returns len(s), nil. +// It may be faster than Write([]byte(s)) by avoiding a copy. +func (d *Digest) WriteString(s string) (n int, err error) { + var b []byte + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data + bh.Len = len(s) + bh.Cap = len(s) + return d.Write(b) +} diff --git a/vendor/github.com/codeskyblue/go-uuid/LICENSE b/vendor/github.com/codeskyblue/go-uuid/LICENSE deleted file mode 100644 index ab6b011a..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/codeskyblue/go-uuid/dce.go b/vendor/github.com/codeskyblue/go-uuid/dce.go deleted file mode 100755 index 50a0f2d0..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/dce.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -import ( - "encoding/binary" - "fmt" - "os" -) - -// A Domain represents a Version 2 domain -type Domain byte - -// Domain constants for DCE Security (Version 2) UUIDs. -const ( - Person = Domain(0) - Group = Domain(1) - Org = Domain(2) -) - -// NewDCESecurity returns a DCE Security (Version 2) UUID. -// -// The domain should be one of Person, Group or Org. -// On a POSIX system the id should be the users UID for the Person -// domain and the users GID for the Group. The meaning of id for -// the domain Org or on non-POSIX systems is site defined. -// -// For a given domain/id pair the same token may be returned for up to -// 7 minutes and 10 seconds. -func NewDCESecurity(domain Domain, id uint32) UUID { - uuid := NewUUID() - if uuid != nil { - uuid[6] = (uuid[6] & 0x0f) | 0x20 // Version 2 - uuid[9] = byte(domain) - binary.BigEndian.PutUint32(uuid[0:], id) - } - return uuid -} - -// NewDCEPerson returns a DCE Security (Version 2) UUID in the person -// domain with the id returned by os.Getuid. -// -// NewDCEPerson(Person, uint32(os.Getuid())) -func NewDCEPerson() UUID { - return NewDCESecurity(Person, uint32(os.Getuid())) -} - -// NewDCEGroup returns a DCE Security (Version 2) UUID in the group -// domain with the id returned by os.Getgid. -// -// NewDCEGroup(Group, uint32(os.Getgid())) -func NewDCEGroup() UUID { - return NewDCESecurity(Group, uint32(os.Getgid())) -} - -// Domain returns the domain for a Version 2 UUID or false. -func (uuid UUID) Domain() (Domain, bool) { - if v, _ := uuid.Version(); v != 2 { - return 0, false - } - return Domain(uuid[9]), true -} - -// Id returns the id for a Version 2 UUID or false. -func (uuid UUID) Id() (uint32, bool) { - if v, _ := uuid.Version(); v != 2 { - return 0, false - } - return binary.BigEndian.Uint32(uuid[0:4]), true -} - -func (d Domain) String() string { - switch d { - case Person: - return "Person" - case Group: - return "Group" - case Org: - return "Org" - } - return fmt.Sprintf("Domain%d", int(d)) -} diff --git a/vendor/github.com/codeskyblue/go-uuid/doc.go b/vendor/github.com/codeskyblue/go-uuid/doc.go deleted file mode 100755 index d8bd013e..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// The uuid package generates and inspects UUIDs. -// -// UUIDs are based on RFC 4122 and DCE 1.1: Authentication and Security Services. -package uuid diff --git a/vendor/github.com/codeskyblue/go-uuid/hash.go b/vendor/github.com/codeskyblue/go-uuid/hash.go deleted file mode 100644 index cdd4192f..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/hash.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -import ( - "crypto/md5" - "crypto/sha1" - "hash" -) - -// Well known Name Space IDs and UUIDs -var ( - NameSpace_DNS = Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - NameSpace_URL = Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8") - NameSpace_OID = Parse("6ba7b812-9dad-11d1-80b4-00c04fd430c8") - NameSpace_X500 = Parse("6ba7b814-9dad-11d1-80b4-00c04fd430c8") - NIL = Parse("00000000-0000-0000-0000-000000000000") -) - -// NewHash returns a new UUID dervied from the hash of space concatenated with -// data generated by h. The hash should be at least 16 byte in length. The -// first 16 bytes of the hash are used to form the UUID. The version of the -// UUID will be the lower 4 bits of version. NewHash is used to implement -// NewMD5 and NewSHA1. -func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID { - h.Reset() - h.Write(space) - h.Write([]byte(data)) - s := h.Sum(nil) - uuid := make([]byte, 16) - copy(uuid, s) - uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4) - uuid[8] = (uuid[8] & 0x3f) | 0x80 // RFC 4122 variant - return uuid -} - -// NewMD5 returns a new MD5 (Version 3) UUID based on the -// supplied name space and data. -// -// NewHash(md5.New(), space, data, 3) -func NewMD5(space UUID, data []byte) UUID { - return NewHash(md5.New(), space, data, 3) -} - -// NewSHA1 returns a new SHA1 (Version 5) UUID based on the -// supplied name space and data. -// -// NewHash(sha1.New(), space, data, 5) -func NewSHA1(space UUID, data []byte) UUID { - return NewHash(sha1.New(), space, data, 5) -} diff --git a/vendor/github.com/codeskyblue/go-uuid/node.go b/vendor/github.com/codeskyblue/go-uuid/node.go deleted file mode 100755 index dd0a8ac1..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/node.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -import "net" - -var ( - interfaces []net.Interface // cached list of interfaces - ifname string // name of interface being used - nodeID []byte // hardware for version 1 UUIDs -) - -// NodeInterface returns the name of the interface from which the NodeID was -// derived. The interface "user" is returned if the NodeID was set by -// SetNodeID. -func NodeInterface() string { - return ifname -} - -// SetNodeInterface selects the hardware address to be used for Version 1 UUIDs. -// If name is "" then the first usable interface found will be used or a random -// Node ID will be generated. If a named interface cannot be found then false -// is returned. -// -// SetNodeInterface never fails when name is "". -func SetNodeInterface(name string) bool { - if interfaces == nil { - var err error - interfaces, err = net.Interfaces() - if err != nil && name != "" { - return false - } - } - - for _, ifs := range interfaces { - if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) { - if setNodeID(ifs.HardwareAddr) { - ifname = ifs.Name - return true - } - } - } - - // We found no interfaces with a valid hardware address. If name - // does not specify a specific interface generate a random Node ID - // (section 4.1.6) - if name == "" { - if nodeID == nil { - nodeID = make([]byte, 6) - } - randomBits(nodeID) - return true - } - return false -} - -// NodeID returns a slice of a copy of the current Node ID, setting the Node ID -// if not already set. -func NodeID() []byte { - if nodeID == nil { - SetNodeInterface("") - } - nid := make([]byte, 6) - copy(nid, nodeID) - return nid -} - -// SetNodeID sets the Node ID to be used for Version 1 UUIDs. The first 6 bytes -// of id are used. If id is less than 6 bytes then false is returned and the -// Node ID is not set. -func SetNodeID(id []byte) bool { - if setNodeID(id) { - ifname = "user" - return true - } - return false -} - -func setNodeID(id []byte) bool { - if len(id) < 6 { - return false - } - if nodeID == nil { - nodeID = make([]byte, 6) - } - copy(nodeID, id) - return true -} - -// NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is -// not valid. The NodeID is only well defined for version 1 and 2 UUIDs. -func (uuid UUID) NodeID() []byte { - if len(uuid) != 16 { - return nil - } - node := make([]byte, 6) - copy(node, uuid[10:]) - return node -} diff --git a/vendor/github.com/codeskyblue/go-uuid/time.go b/vendor/github.com/codeskyblue/go-uuid/time.go deleted file mode 100755 index ad467968..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/time.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -import ( - "encoding/binary" - "time" -) - -// A Time represents a time as the number of 100's of nanoseconds since 15 Oct -// 1582. -type Time int64 - -const ( - lillian = 2299160 // Julian day of 15 Oct 1582 - unix = 2440587 // Julian day of 1 Jan 1970 - epoch = unix - lillian // Days between epochs - g1582 = epoch * 86400 // seconds between epochs - g1582ns100 = g1582 * 10000000 // 100s of a nanoseconds between epochs -) - -var ( - lasttime uint64 // last time we returned - clock_seq uint16 // clock sequence for this run - - timeNow = time.Now // for testing -) - -// UnixTime converts t the number of seconds and nanoseconds using the Unix -// epoch of 1 Jan 1970. -func (t Time) UnixTime() (sec, nsec int64) { - sec = int64(t - g1582ns100) - nsec = (sec % 10000000) * 100 - sec /= 10000000 - return sec, nsec -} - -// GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and -// adjusts the clock sequence as needed. An error is returned if the current -// time cannot be determined. -func GetTime() (Time, error) { - t := timeNow() - - // If we don't have a clock sequence already, set one. - if clock_seq == 0 { - SetClockSequence(-1) - } - now := uint64(t.UnixNano()/100) + g1582ns100 - - // If time has gone backwards with this clock sequence then we - // increment the clock sequence - if now <= lasttime { - clock_seq = ((clock_seq + 1) & 0x3fff) | 0x8000 - } - lasttime = now - return Time(now), nil -} - -// ClockSequence returns the current clock sequence, generating one if not -// already set. The clock sequence is only used for Version 1 UUIDs. -// -// The uuid package does not use global static storage for the clock sequence or -// the last time a UUID was generated. Unless SetClockSequence a new random -// clock sequence is generated the first time a clock sequence is requested by -// ClockSequence, GetTime, or NewUUID. (section 4.2.1.1) sequence is generated -// for -func ClockSequence() int { - if clock_seq == 0 { - SetClockSequence(-1) - } - return int(clock_seq & 0x3fff) -} - -// SetClockSeq sets the clock sequence to the lower 14 bits of seq. Setting to -// -1 causes a new sequence to be generated. -func SetClockSequence(seq int) { - if seq == -1 { - var b [2]byte - randomBits(b[:]) // clock sequence - seq = int(b[0])<<8 | int(b[1]) - } - old_seq := clock_seq - clock_seq = uint16(seq&0x3fff) | 0x8000 // Set our variant - if old_seq != clock_seq { - lasttime = 0 - } -} - -// Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in -// uuid. It returns false if uuid is not valid. The time is only well defined -// for version 1 and 2 UUIDs. -func (uuid UUID) Time() (Time, bool) { - if len(uuid) != 16 { - return 0, false - } - time := int64(binary.BigEndian.Uint32(uuid[0:4])) - time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 - time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 - return Time(time), true -} - -// ClockSequence returns the clock sequence encoded in uuid. It returns false -// if uuid is not valid. The clock sequence is only well defined for version 1 -// and 2 UUIDs. -func (uuid UUID) ClockSequence() (int, bool) { - if len(uuid) != 16 { - return 0, false - } - return int(binary.BigEndian.Uint16(uuid[8:10])) & 0x3fff, true -} diff --git a/vendor/github.com/codeskyblue/go-uuid/util.go b/vendor/github.com/codeskyblue/go-uuid/util.go deleted file mode 100644 index de40b102..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/util.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -import ( - "io" -) - -// randomBits completely fills slice b with random data. -func randomBits(b []byte) { - if _, err := io.ReadFull(rander, b); err != nil { - panic(err.Error()) // rand should never fail - } -} - -// xvalues returns the value of a byte as a hexadecimal digit or 255. -var xvalues = []byte{ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, - 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, -} - -// xtob converts the the first two hex bytes of x into a byte. -func xtob(x string) (byte, bool) { - b1 := xvalues[x[0]] - b2 := xvalues[x[1]] - return (b1 << 4) | b2, b1 != 255 && b2 != 255 -} diff --git a/vendor/github.com/codeskyblue/go-uuid/uuid.go b/vendor/github.com/codeskyblue/go-uuid/uuid.go deleted file mode 100755 index 2920fae6..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/uuid.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -import ( - "bytes" - "crypto/rand" - "fmt" - "io" - "strings" -) - -// A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC -// 4122. -type UUID []byte - -// A Version represents a UUIDs version. -type Version byte - -// A Variant represents a UUIDs variant. -type Variant byte - -// Constants returned by Variant. -const ( - Invalid = Variant(iota) // Invalid UUID - RFC4122 // The variant specified in RFC4122 - Reserved // Reserved, NCS backward compatibility. - Microsoft // Reserved, Microsoft Corporation backward compatibility. - Future // Reserved for future definition. -) - -var rander = rand.Reader // random function - -// New returns a new random (version 4) UUID as a string. It is a convenience -// function for NewRandom().String(). -func New() string { - return NewRandom().String() -} - -// Parse decodes s into a UUID or returns nil. Both the UUID form of -// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and -// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded. -func Parse(s string) UUID { - if len(s) == 36+9 { - if strings.ToLower(s[:9]) != "urn:uuid:" { - return nil - } - s = s[9:] - } else if len(s) != 36 { - return nil - } - if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { - return nil - } - uuid := make([]byte, 16) - for i, x := range []int{ - 0, 2, 4, 6, - 9, 11, - 14, 16, - 19, 21, - 24, 26, 28, 30, 32, 34} { - if v, ok := xtob(s[x:]); !ok { - return nil - } else { - uuid[i] = v - } - } - return uuid -} - -// Equal returns true if uuid1 and uuid2 are equal. -func Equal(uuid1, uuid2 UUID) bool { - return bytes.Equal(uuid1, uuid2) -} - -// String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -// , or "" if uuid is invalid. -func (uuid UUID) String() string { - if uuid == nil || len(uuid) != 16 { - return "" - } - b := []byte(uuid) - return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", - b[:4], b[4:6], b[6:8], b[8:10], b[10:]) -} - -// URN returns the RFC 2141 URN form of uuid, -// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, or "" if uuid is invalid. -func (uuid UUID) URN() string { - if uuid == nil || len(uuid) != 16 { - return "" - } - b := []byte(uuid) - return fmt.Sprintf("urn:uuid:%08x-%04x-%04x-%04x-%012x", - b[:4], b[4:6], b[6:8], b[8:10], b[10:]) -} - -// Variant returns the variant encoded in uuid. It returns Invalid if -// uuid is invalid. -func (uuid UUID) Variant() Variant { - if len(uuid) != 16 { - return Invalid - } - switch { - case (uuid[8] & 0xc0) == 0x80: - return RFC4122 - case (uuid[8] & 0xe0) == 0xc0: - return Microsoft - case (uuid[8] & 0xe0) == 0xe0: - return Future - default: - return Reserved - } - panic("unreachable") -} - -// Version returns the verison of uuid. It returns false if uuid is not -// valid. -func (uuid UUID) Version() (Version, bool) { - if len(uuid) != 16 { - return 0, false - } - return Version(uuid[6] >> 4), true -} - -func (v Version) String() string { - if v > 15 { - return fmt.Sprintf("BAD_VERSION_%d", v) - } - return fmt.Sprintf("VERSION_%d", v) -} - -func (v Variant) String() string { - switch v { - case RFC4122: - return "RFC4122" - case Reserved: - return "Reserved" - case Microsoft: - return "Microsoft" - case Future: - return "Future" - case Invalid: - return "Invalid" - } - return fmt.Sprintf("BadVariant%d", int(v)) -} - -// SetRand sets the random number generator to r, which implents io.Reader. -// If r.Read returns an error when the package requests random data then -// a panic will be issued. -// -// Calling SetRand with nil sets the random number generator to the default -// generator. -func SetRand(r io.Reader) { - if r == nil { - rander = rand.Reader - return - } - rander = r -} diff --git a/vendor/github.com/codeskyblue/go-uuid/version1.go b/vendor/github.com/codeskyblue/go-uuid/version1.go deleted file mode 100644 index 63580044..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/version1.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -import ( - "encoding/binary" -) - -// NewUUID returns a Version 1 UUID based on the current NodeID and clock -// sequence, and the current time. If the NodeID has not been set by SetNodeID -// or SetNodeInterface then it will be set automatically. If the NodeID cannot -// be set NewUUID returns nil. If clock sequence has not been set by -// SetClockSequence then it will be set automatically. If GetTime fails to -// return the current NewUUID returns nil. -func NewUUID() UUID { - if nodeID == nil { - SetNodeInterface("") - } - - now, err := GetTime() - if err != nil { - return nil - } - - uuid := make([]byte, 16) - - time_low := uint32(now & 0xffffffff) - time_mid := uint16((now >> 32) & 0xffff) - time_hi := uint16((now >> 48) & 0x0fff) - time_hi |= 0x1000 // Version 1 - - binary.BigEndian.PutUint32(uuid[0:], time_low) - binary.BigEndian.PutUint16(uuid[4:], time_mid) - binary.BigEndian.PutUint16(uuid[6:], time_hi) - binary.BigEndian.PutUint16(uuid[8:], clock_seq) - copy(uuid[10:], nodeID) - - return uuid -} diff --git a/vendor/github.com/codeskyblue/go-uuid/version4.go b/vendor/github.com/codeskyblue/go-uuid/version4.go deleted file mode 100644 index b3d4a368..00000000 --- a/vendor/github.com/codeskyblue/go-uuid/version4.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package uuid - -// Random returns a Random (Version 4) UUID or panics. -// -// The strength of the UUIDs is based on the strength of the crypto/rand -// package. -// -// A note about uniqueness derived from from the UUID Wikipedia entry: -// -// Randomly generated UUIDs have 122 random bits. One's annual risk of being -// hit by a meteorite is estimated to be one chance in 17 billion, that -// means the probability is about 0.00000000006 (6 × 10−11), -// equivalent to the odds of creating a few tens of trillions of UUIDs in a -// year and having one duplicate. -func NewRandom() UUID { - uuid := make([]byte, 16) - randomBits([]byte(uuid)) - uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4 - uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 - return uuid -} diff --git a/vendor/github.com/dgryski/go-jump/LICENSE b/vendor/github.com/dgryski/go-jump/LICENSE new file mode 100644 index 00000000..45eb9c0d --- /dev/null +++ b/vendor/github.com/dgryski/go-jump/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Damian Gryski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/dgryski/go-jump/jump.go b/vendor/github.com/dgryski/go-jump/jump.go new file mode 100644 index 00000000..558ae6b9 --- /dev/null +++ b/vendor/github.com/dgryski/go-jump/jump.go @@ -0,0 +1,23 @@ +// Package jump implements Google's Jump Consistent Hash +/* + +From the paper "A Fast, Minimal Memory, Consistent Hash Algorithm" by John Lamping, Eric Veach (2014). + +http://arxiv.org/abs/1406.2294 +*/ +package jump + +// Hash consistently chooses a hash bucket number in the range [0, numBuckets) for the given key. numBuckets must be >= 1. +func Hash(key uint64, numBuckets int) int32 { + + var b int64 = -1 + var j int64 + + for j < int64(numBuckets) { + b = j + key = key*2862933555777941757 + 1 + j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1))) + } + + return int32(b) +} diff --git a/vendor/github.com/raintank/metrictank/LICENSE b/vendor/github.com/grafana/metrictank/LICENSE similarity index 100% rename from vendor/github.com/raintank/metrictank/LICENSE rename to vendor/github.com/grafana/metrictank/LICENSE diff --git a/vendor/github.com/raintank/metrictank/NOTICE b/vendor/github.com/grafana/metrictank/NOTICE similarity index 86% rename from vendor/github.com/raintank/metrictank/NOTICE rename to vendor/github.com/grafana/metrictank/NOTICE index 8002a802..ea4da4a3 100644 --- a/vendor/github.com/raintank/metrictank/NOTICE +++ b/vendor/github.com/grafana/metrictank/NOTICE @@ -1,4 +1,4 @@ -Copyright 2016 Dieter Plaetinck, Anthony Woods, Jeremy Bingham, Damian Gryski, raintank inc +Copyright 2016-2018 Dieter Plaetinck, Anthony Woods, Jeremy Bingham, Damian Gryski, raintank inc Metrictank is free software: you can redistribute it and/or modify diff --git a/vendor/github.com/grafana/metrictank/batch/LICENSE.TXT b/vendor/github.com/grafana/metrictank/batch/LICENSE.TXT new file mode 100644 index 00000000..4b1ae4e4 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/batch/LICENSE.TXT @@ -0,0 +1,14 @@ +Copyright 2018 Grafana Labs, Bloomberg + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use the files in this directory +except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/github.com/grafana/metrictank/cluster/partitioner/partitioner.go b/vendor/github.com/grafana/metrictank/cluster/partitioner/partitioner.go new file mode 100644 index 00000000..b9870ee7 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/cluster/partitioner/partitioner.go @@ -0,0 +1,34 @@ +package partitioner + +import ( + "fmt" + + "github.com/grafana/metrictank/schema" +) + +type Partitioner interface { + Partition(schema.PartitionedMetric, int32) (int32, error) +} + +type Kafka struct { + Method schema.PartitionByMethod +} + +func NewKafka(partitionBy string) (*Kafka, error) { + var method schema.PartitionByMethod + switch partitionBy { + case "byOrg": + method = schema.PartitionByOrg + case "bySeries": + method = schema.PartitionBySeries + case "bySeriesWithTags": + method = schema.PartitionBySeriesWithTags + default: + return nil, fmt.Errorf("partitionBy must be one of 'byOrg|bySeries|bySeriesWithTags'. got %s", partitionBy) + } + return &Kafka{Method: method}, nil +} + +func (k *Kafka) Partition(m schema.PartitionedMetric, numPartitions int32) (int32, error) { + return m.PartitionID(k.Method, numPartitions) +} diff --git a/vendor/github.com/grafana/metrictank/consolidation/LICENSE.TXT b/vendor/github.com/grafana/metrictank/consolidation/LICENSE.TXT new file mode 100644 index 00000000..3693aa72 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/consolidation/LICENSE.TXT @@ -0,0 +1,14 @@ +Copyright 2018 Grafana Labs + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use the files in this directory +except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/github.com/grafana/metrictank/dashboards/dockprom/LICENSE b/vendor/github.com/grafana/metrictank/dashboards/dockprom/LICENSE new file mode 100644 index 00000000..cd9083bd --- /dev/null +++ b/vendor/github.com/grafana/metrictank/dashboards/dockprom/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Stefan Prodan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/grafana/metrictank/expr/LICENSE b/vendor/github.com/grafana/metrictank/expr/LICENSE new file mode 100644 index 00000000..edffca4c --- /dev/null +++ b/vendor/github.com/grafana/metrictank/expr/LICENSE @@ -0,0 +1,26 @@ +Regarding type expr, much of parse.go, parse_test.go: + +Copyright (c) 2017-2018 Grafana Labs +Copyright (c) 2014,2015 Damian Gryski +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/grafana/metrictank/mdata/chunk/tsz/LICENSE b/vendor/github.com/grafana/metrictank/mdata/chunk/tsz/LICENSE new file mode 100644 index 00000000..97b198a4 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/mdata/chunk/tsz/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2015,2016 Damian Gryski +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/grafana/metrictank/schema/archive.go b/vendor/github.com/grafana/metrictank/schema/archive.go new file mode 100644 index 00000000..9287a9c1 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/archive.go @@ -0,0 +1,117 @@ +package schema + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +//go:generate stringer -type=Method -linecomment +//go:generate msgp +//msgp:ignore Method + +// Archive represents a metric archive +// the zero value represents a raw metric +// any non-zero value represents a certain +// aggregation method (lower 8 bits) and +// aggregation span (higher 8 bits) +type Archive uint16 + +// important: caller must make sure to call IsSpanValid first +func NewArchive(method Method, span uint32) Archive { + code := uint16(spanHumanToCode[span]) + return Archive(uint16(method) | code<<8) +} + +func ArchiveFromString(s string) (Archive, error) { + pos := strings.Index(s, "_") + if pos == -1 { + return 0, ErrInvalidFormat + } + + method, err := MethodFromString(s[:pos]) + if err != nil { + return 0, err + } + span, err := strconv.ParseInt(s[pos+1:], 10, 32) + if err != nil { + return 0, err + } + if !IsSpanValid(uint32(span)) { + return 0, fmt.Errorf("invalid span %d", span) + } + return NewArchive(method, uint32(span)), nil +} + +// String returns the traditional key suffix like sum_600 etc +// (invalid to call this for raw archives) +func (a Archive) String() string { + return a.Method().String() + "_" + strconv.FormatInt(int64(a.Span()), 10) +} + +func (a Archive) Method() Method { + return Method(a & 0x0F) +} + +func (a Archive) Span() uint32 { + if a == 0 { + return 0 + } + return spanCodeToHuman[uint8(a>>8)] +} + +func IsSpanValid(span uint32) bool { + _, ok := spanHumanToCode[span] + return ok +} + +type Method uint8 + +const ( + Avg Method = iota + 1 // avg + Sum // sum + Lst // lst + Max // max + Min // min + Cnt // cnt +) + +func MethodFromString(input string) (Method, error) { + switch input { + case "avg": + return Avg, nil + case "sum": + return Sum, nil + case "lst": + return Lst, nil + case "max": + return Max, nil + case "min": + return Min, nil + case "cnt": + return Cnt, nil + } + return 0, errors.New("no such method") +} + +// maps human friendly span numbers (in seconds) to optimized code form +var spanHumanToCode map[uint32]uint8 + +// maps span codes to human friendly span numbers in seconds +var spanCodeToHuman map[uint8]uint32 + +func init() { + // all the aggregation spans we support, their index position in this slice is their code + spans := []uint32{2, 5, 10, 15, 30, 60, 90, 120, 150, 300, 600, 900, 1200, 1800, 45 * 60, 3600, 3600 + 30*60, 2 * 3600, 3 * 3600, 4 * 3600, 5 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, 24 * 3600} + + spanHumanToCode = make(map[uint32]uint8) + spanCodeToHuman = make(map[uint8]uint32) + + for i, human := range spans { + code := uint8(i) + spanHumanToCode[human] = code + spanCodeToHuman[code] = human + } + +} diff --git a/vendor/github.com/grafana/metrictank/schema/archive_gen.go b/vendor/github.com/grafana/metrictank/schema/archive_gen.go new file mode 100644 index 00000000..a5c87db4 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/archive_gen.go @@ -0,0 +1,59 @@ +package schema + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *Archive) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint16 + zb0001, err = dc.ReadUint16() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Archive(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Archive) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint16(uint16(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Archive) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint16(o, uint16(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Archive) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint16 + zb0001, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Archive(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Archive) Msgsize() (s int) { + s = msgp.Uint16Size + return +} diff --git a/vendor/github.com/grafana/metrictank/schema/key.go b/vendor/github.com/grafana/metrictank/schema/key.go new file mode 100644 index 00000000..a6b7f801 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/key.go @@ -0,0 +1,103 @@ +package schema + +import ( + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" +) + +//go:generate msgp +//msgp:ignore AMKey +// don't ignore Key, MKey because it's used for MetricDefinition + +var ErrStringTooShort = errors.New("string too short") +var ErrInvalidFormat = errors.New("invalid format") + +// Key identifies a metric +type Key [16]byte + +// MKey uniquely identifies a metric in a multi-tenant context +type MKey struct { + Key Key + Org uint32 +} + +// KeyFromString parses a string id to an MKey +// string id must be of form orgid. +func MKeyFromString(s string) (MKey, error) { + l := len(s) + + // shortest an orgid can be is single digit + if l < 34 { + return MKey{}, ErrStringTooShort + } + + hashStr := s[l-32:] + orgStr := s[0 : l-33] + + hash, err := hex.DecodeString(hashStr) + if err != nil { + return MKey{}, err + } + + org, err := strconv.ParseUint(orgStr, 10, 32) + if err != nil { + return MKey{}, err + } + + k := MKey{ + Org: uint32(org), + } + + copy(k.Key[:], hash) + return k, nil +} + +func (m MKey) String() string { + return fmt.Sprintf("%d.%x", m.Org, m.Key) +} + +// AMKey is a multi-tenant key with archive extension +// so you can refer to rollup archives +type AMKey struct { + MKey MKey + Archive Archive +} + +func (a AMKey) String() string { + if a.Archive == 0 { + return a.MKey.String() + } + return a.MKey.String() + "_" + a.Archive.String() +} + +// GetAMKey helps to easily get an AMKey from a given MKey +func GetAMKey(m MKey, method Method, span uint32) AMKey { + return AMKey{ + MKey: m, + Archive: NewArchive(method, span), + } +} + +func AMKeyFromString(s string) (AMKey, error) { + underscores := strings.Count(s, "_") + amk := AMKey{} + var err error + switch underscores { + case 0: + amk.MKey, err = MKeyFromString(s) + return amk, err + case 2: + pos := strings.Index(s, "_") + amk.MKey, err = MKeyFromString(s[:pos]) + if err != nil { + return amk, err + } + amk.Archive, err = ArchiveFromString(s[pos+1:]) + return amk, err + + } + return amk, ErrInvalidFormat +} diff --git a/vendor/github.com/grafana/metrictank/schema/key_gen.go b/vendor/github.com/grafana/metrictank/schema/key_gen.go new file mode 100644 index 00000000..a1bbb608 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/key_gen.go @@ -0,0 +1,179 @@ +package schema + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *Key) DecodeMsg(dc *msgp.Reader) (err error) { + err = dc.ReadExactBytes((z)[:]) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *Key) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteBytes((z)[:]) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *Key) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendBytes(o, (z)[:]) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Key) UnmarshalMsg(bts []byte) (o []byte, err error) { + bts, err = msgp.ReadExactBytes(bts, (z)[:]) + if err != nil { + err = msgp.WrapError(err) + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Key) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *MKey) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Key": + err = dc.ReadExactBytes((z.Key)[:]) + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + case "Org": + z.Org, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Org") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *MKey) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Key" + err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteBytes((z.Key)[:]) + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + // write "Org" + err = en.Append(0xa3, 0x4f, 0x72, 0x67) + if err != nil { + return + } + err = en.WriteUint32(z.Org) + if err != nil { + err = msgp.WrapError(err, "Org") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MKey) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Key" + o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79) + o = msgp.AppendBytes(o, (z.Key)[:]) + // string "Org" + o = append(o, 0xa3, 0x4f, 0x72, 0x67) + o = msgp.AppendUint32(o, z.Org) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MKey) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Key": + bts, err = msgp.ReadExactBytes(bts, (z.Key)[:]) + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + case "Org": + z.Org, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Org") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MKey) Msgsize() (s int) { + s = 1 + 4 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 4 + msgp.Uint32Size + return +} diff --git a/vendor/github.com/grafana/metrictank/schema/method_string.go b/vendor/github.com/grafana/metrictank/schema/method_string.go new file mode 100644 index 00000000..5fe6c345 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/method_string.go @@ -0,0 +1,29 @@ +// Code generated by "stringer -type=Method -linecomment"; DO NOT EDIT. + +package schema + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Avg-1] + _ = x[Sum-2] + _ = x[Lst-3] + _ = x[Max-4] + _ = x[Min-5] + _ = x[Cnt-6] +} + +const _Method_name = "avgsumlstmaxmincnt" + +var _Method_index = [...]uint8{0, 3, 6, 9, 12, 15, 18} + +func (i Method) String() string { + i -= 1 + if i >= Method(len(_Method_index)-1) { + return "Method(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _Method_name[_Method_index[i]:_Method_index[i+1]] +} diff --git a/vendor/github.com/grafana/metrictank/schema/metric.go b/vendor/github.com/grafana/metrictank/schema/metric.go new file mode 100644 index 00000000..b3a06da4 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/metric.go @@ -0,0 +1,391 @@ +package schema + +import ( + "bytes" + "crypto/md5" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync/atomic" +) + +var ErrInvalidIntervalzero = errors.New("interval cannot be 0") +var ErrInvalidOrgIdzero = errors.New("org-id cannot be 0") +var ErrInvalidEmptyName = errors.New("name cannot be empty") +var ErrInvalidMtype = errors.New("invalid mtype") +var ErrInvalidTagFormat = errors.New("invalid tag format") +var ErrUnknownPartitionMethod = errors.New("unknown partition method") + +type PartitionedMetric interface { + Validate() error + SetId() + // PartitionID returns the partition id that should be used for this metric. + PartitionID(method PartitionByMethod, partitions int32) (int32, error) +} + +//go:generate msgp + +// MetricData contains all metric metadata (some as fields, some as tags) and a datapoint +type MetricData struct { + Id string `json:"id"` + OrgId int `json:"org_id"` + Name string `json:"name"` + Interval int `json:"interval"` + Value float64 `json:"value"` + Unit string `json:"unit"` + Time int64 `json:"time"` + Mtype string `json:"mtype"` + Tags []string `json:"tags"` +} + +func (m *MetricData) Validate() error { + if m.OrgId == 0 { + return ErrInvalidOrgIdzero + } + if m.Interval == 0 { + return ErrInvalidIntervalzero + } + m.Name = EatDots(m.Name) + if m.Name == "" { + return ErrInvalidEmptyName + } + if m.Mtype == "" || (m.Mtype != "gauge" && m.Mtype != "rate" && m.Mtype != "count" && m.Mtype != "counter" && m.Mtype != "timestamp") { + return ErrInvalidMtype + } + if !ValidateTags(m.Tags) { + return ErrInvalidTagFormat + } + return nil +} + +// returns a id (hash key) in the format OrgId.md5Sum +// the md5sum is a hash of the the concatenation of the +// metric + each tag key:value pair (in metrics2.0 sense, so also fields), sorted alphabetically. +func (m *MetricData) SetId() { + sort.Strings(m.Tags) + + buffer := bytes.NewBufferString(m.Name) + buffer.WriteByte(0) + buffer.WriteString(m.Unit) + buffer.WriteByte(0) + buffer.WriteString(m.Mtype) + buffer.WriteByte(0) + fmt.Fprintf(buffer, "%d", m.Interval) + + for _, k := range m.Tags { + buffer.WriteByte(0) + buffer.WriteString(k) + } + m.Id = fmt.Sprintf("%d.%x", m.OrgId, md5.Sum(buffer.Bytes())) +} + +// can be used by some encoders, such as msgp +type MetricDataArray []*MetricData + +type MetricDefinition struct { + Id MKey `json:"mkey"` + OrgId uint32 `json:"org_id"` + Name string `json:"name"` + Interval int `json:"interval"` + Unit string `json:"unit"` + Mtype string `json:"mtype"` + + // some users of MetricDefinition (f.e. MetricTank) might add a "name" tag + // to this slice which allows querying by name as a tag. this special tag + // should not be stored or transmitted over the network, otherwise it may + // just get overwritten by the receiver. + Tags []string `json:"tags"` + LastUpdate int64 `json:"lastUpdate"` // unix timestamp + Partition int32 `json:"partition"` + + // this is a special attribute that does not need to be set, it is only used + // to keep the state of NameWithTags() + nameWithTags string +} + +// NameWithTags deduplicates the name and tags strings by storing their content +// as a single string in .nameWithTags and then makes .Name and the .Tags slices +// of it. Once this has been done it won't do it a second time, but just reuse +// the already generated .nameWithTags. +// It returns the deduplicated name with tags. +func (m *MetricDefinition) NameWithTags() string { + if len(m.nameWithTags) > 0 { + return m.nameWithTags + } + + nameWithTagsBuffer := &bytes.Buffer{} + _ = writeSortedTagString(nameWithTagsBuffer, m.Name, m.Tags) + m.nameWithTags = nameWithTagsBuffer.String() + + var i int + cursor := len(m.Name) + m.Name = m.nameWithTags[:cursor] + for _, t := range m.Tags { + if len(t) > 5 && t[:5] == "name=" { + continue + } + m.Tags[i] = m.nameWithTags[cursor+1 : cursor+1+len(t)] + cursor += len(t) + 1 + i++ + } + + // if a "name" tag existed, then we have to shorten the slice + if i < len(m.Tags) { + m.Tags = m.Tags[:i] + } + + return m.nameWithTags +} + +// Clone() returns a copy of the MetricDefinition. It uses atomic operations +// to read certain properties that get updated atomically +func (m *MetricDefinition) Clone() MetricDefinition { + return MetricDefinition{ + Id: m.Id, + OrgId: m.OrgId, + Name: m.Name, + Interval: m.Interval, + Unit: m.Unit, + Mtype: m.Mtype, + Tags: m.Tags, + LastUpdate: atomic.LoadInt64(&m.LastUpdate), + Partition: atomic.LoadInt32(&m.Partition), + nameWithTags: m.nameWithTags, + } +} + +func (m *MetricDefinition) NameSanitizedAsTagValue() string { + return SanitizeNameAsTagValue(m.Name) +} + +func (m *MetricDefinition) SetId() { + sort.Strings(m.Tags) + + buffer := bytes.NewBufferString(m.Name) + buffer.WriteByte(0) + buffer.WriteString(m.Unit) + buffer.WriteByte(0) + buffer.WriteString(m.Mtype) + buffer.WriteByte(0) + fmt.Fprintf(buffer, "%d", m.Interval) + + for _, t := range m.Tags { + if len(t) > 5 && t[:5] == "name=" { + continue + } + + buffer.WriteByte(0) + buffer.WriteString(t) + } + + m.Id = MKey{ + md5.Sum(buffer.Bytes()), + uint32(m.OrgId), + } +} + +func (m *MetricDefinition) Validate() error { + if m.OrgId == 0 { + return ErrInvalidOrgIdzero + } + if m.Interval == 0 { + return ErrInvalidIntervalzero + } + m.Name = EatDots(m.Name) + if m.Name == "" { + return ErrInvalidEmptyName + } + if m.Mtype == "" || (m.Mtype != "gauge" && m.Mtype != "rate" && m.Mtype != "count" && m.Mtype != "counter" && m.Mtype != "timestamp") { + return ErrInvalidMtype + } + if !ValidateTags(m.Tags) { + return ErrInvalidTagFormat + } + return nil +} + +// MetricDefinitionFromMetricData yields a MetricDefinition that has no references +// to the original MetricData +func MetricDefinitionFromMetricData(d *MetricData) *MetricDefinition { + tags := make([]string, len(d.Tags)) + copy(tags, d.Tags) + mkey, _ := MKeyFromString(d.Id) + + md := &MetricDefinition{ + Id: mkey, + Name: d.Name, + OrgId: uint32(d.OrgId), + Mtype: d.Mtype, + Interval: d.Interval, + LastUpdate: d.Time, + Unit: d.Unit, + Tags: tags, + } + + return md +} + +// SanitizeNameAsTagValue takes a name and potentially +// modifies it to ensure it is a valid value that can be +// used as a tag value. This is important when we index +// metric names as values of the tag "name" +func SanitizeNameAsTagValue(name string) string { + if len(name) == 0 || name[0] != '~' { + return name + } + + for i := 1; i < len(name); i++ { + if name[i] != '~' { + return name[i:] + } + } + + // the whole name consists of no other chars than '~' + return "" +} + +// EatDots removes multiple consecutive, leading, and trailing dots +// from name. If the provided name is only dots, it will return an +// empty string +// The vast majority of names will not need to be modified, +// so we optimize for that case. This function only requires +// allocations if the name does need to be modified. +func EatDots(name string) string { + if len(name) == 0 { + return "" + } + + dotsToRemove := 0 + if name[0] == '.' { + dotsToRemove++ + } + for i := 1; i < len(name); i++ { + if name[i] == '.' { + if name[i-1] == '.' { + dotsToRemove++ + } + if i == len(name)-1 { + dotsToRemove++ + } + } + } + + // the majority of cases will return here + if dotsToRemove == 0 { + return name + } + + if dotsToRemove >= len(name) { + return "" + } + + newName := make([]byte, len(name)-dotsToRemove) + j := 0 + sawDot := false + for i := 0; i < len(name); i++ { + if name[i] == '.' { + if j > 0 { + sawDot = true + } + continue + } + + if sawDot { + newName[j] = '.' + sawDot = false + j++ + } + + newName[j] = name[i] + j++ + } + + return string(newName) +} + +// ValidateTags returns whether all tags are in a valid format. +// a valid format is anything that looks like key=value, +// the length of key and value must be >0 and both cannot contain +// the certain prohibited characters +func ValidateTags(tags []string) bool { + for _, t := range tags { + if !ValidateTag(t) { + return false + } + } + + return true +} + +func ValidateTag(tag string) bool { + // a valid tag must have: + // - a key that's at least 1 char long + // - a = sign + // - a value that's at least 1 char long + if len(tag) < 3 { + return false + } + + equal := strings.Index(tag, "=") + if equal == -1 { + return false + } + + // first equal sign must not be the first nor last character + if equal == 0 || equal == len(tag)-1 { + return false + } + + return ValidateTagKey(tag[:equal]) && ValidateTagValue(tag[equal+1:]) +} + +// ValidateTagKey validates tag key requirements as defined in graphite docs +func ValidateTagKey(key string) bool { + if len(key) == 0 { + return false + } + + return !strings.ContainsAny(key, ";!^=") +} + +// ValidateTagValue is the same as the above ValidateTagKey, but for the tag value +func ValidateTagValue(value string) bool { + if len(value) == 0 { + return false + } + + if value[0] == '~' { + return false + } + + return !strings.ContainsRune(value, ';') +} + +func writeSortedTagString(w io.Writer, name string, tags []string) error { + sort.Strings(tags) + + _, err := io.WriteString(w, name) + if err != nil { + return err + } + + for _, t := range tags { + if len(t) > 5 && t[:5] == "name=" { + continue + } + + _, err = io.WriteString(w, ";") + if err != nil { + return err + } + + _, err = io.WriteString(w, t) + if err != nil { + return err + } + } + + return nil +} diff --git a/vendor/gopkg.in/raintank/schema.v1/metric_gen.go b/vendor/github.com/grafana/metrictank/schema/metric_gen.go similarity index 63% rename from vendor/gopkg.in/raintank/schema.v1/metric_gen.go rename to vendor/github.com/grafana/metrictank/schema/metric_gen.go index d0f17dc7..0d793ac3 100644 --- a/vendor/gopkg.in/raintank/schema.v1/metric_gen.go +++ b/vendor/github.com/grafana/metrictank/schema/metric_gen.go @@ -1,92 +1,100 @@ package schema -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT +// Code generated by github.com/tinylib/msgp DO NOT EDIT. -import "github.com/tinylib/msgp/msgp" +import ( + "github.com/tinylib/msgp/msgp" +) // DecodeMsg implements msgp.Decodable func (z *MetricData) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field - var zbzg uint32 - zbzg, err = dc.ReadMapHeader() + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err) return } - for zbzg > 0 { - zbzg-- + for zb0001 > 0 { + zb0001-- field, err = dc.ReadMapKeyPtr() if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Id": z.Id, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Id") return } case "OrgId": z.OrgId, err = dc.ReadInt() if err != nil { + err = msgp.WrapError(err, "OrgId") return } case "Name": z.Name, err = dc.ReadString() if err != nil { - return - } - case "Metric": - z.Metric, err = dc.ReadString() - if err != nil { + err = msgp.WrapError(err, "Name") return } case "Interval": z.Interval, err = dc.ReadInt() if err != nil { + err = msgp.WrapError(err, "Interval") return } case "Value": z.Value, err = dc.ReadFloat64() if err != nil { + err = msgp.WrapError(err, "Value") return } case "Unit": z.Unit, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Unit") return } case "Time": z.Time, err = dc.ReadInt64() if err != nil { + err = msgp.WrapError(err, "Time") return } case "Mtype": z.Mtype, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Mtype") return } case "Tags": - var zbai uint32 - zbai, err = dc.ReadArrayHeader() + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err, "Tags") return } - if cap(z.Tags) >= int(zbai) { - z.Tags = (z.Tags)[:zbai] + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] } else { - z.Tags = make([]string, zbai) + z.Tags = make([]string, zb0002) } - for zxvk := range z.Tags { - z.Tags[zxvk], err = dc.ReadString() + for za0001 := range z.Tags { + z.Tags[za0001], err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Tags", za0001) return } } default: err = dc.Skip() if err != nil { + err = msgp.WrapError(err) return } } @@ -96,100 +104,101 @@ func (z *MetricData) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *MetricData) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 10 + // map header, size 9 // write "Id" - err = en.Append(0x8a, 0xa2, 0x49, 0x64) + err = en.Append(0x89, 0xa2, 0x49, 0x64) if err != nil { - return err + return } err = en.WriteString(z.Id) if err != nil { + err = msgp.WrapError(err, "Id") return } // write "OrgId" err = en.Append(0xa5, 0x4f, 0x72, 0x67, 0x49, 0x64) if err != nil { - return err + return } err = en.WriteInt(z.OrgId) if err != nil { + err = msgp.WrapError(err, "OrgId") return } // write "Name" err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65) - if err != nil { - return err - } - err = en.WriteString(z.Name) if err != nil { return } - // write "Metric" - err = en.Append(0xa6, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63) - if err != nil { - return err - } - err = en.WriteString(z.Metric) + err = en.WriteString(z.Name) if err != nil { + err = msgp.WrapError(err, "Name") return } // write "Interval" err = en.Append(0xa8, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c) if err != nil { - return err + return } err = en.WriteInt(z.Interval) if err != nil { + err = msgp.WrapError(err, "Interval") return } // write "Value" err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65) if err != nil { - return err + return } err = en.WriteFloat64(z.Value) if err != nil { + err = msgp.WrapError(err, "Value") return } // write "Unit" err = en.Append(0xa4, 0x55, 0x6e, 0x69, 0x74) if err != nil { - return err + return } err = en.WriteString(z.Unit) if err != nil { + err = msgp.WrapError(err, "Unit") return } // write "Time" err = en.Append(0xa4, 0x54, 0x69, 0x6d, 0x65) if err != nil { - return err + return } err = en.WriteInt64(z.Time) if err != nil { + err = msgp.WrapError(err, "Time") return } // write "Mtype" err = en.Append(0xa5, 0x4d, 0x74, 0x79, 0x70, 0x65) if err != nil { - return err + return } err = en.WriteString(z.Mtype) if err != nil { + err = msgp.WrapError(err, "Mtype") return } // write "Tags" err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73) if err != nil { - return err + return } err = en.WriteArrayHeader(uint32(len(z.Tags))) if err != nil { + err = msgp.WrapError(err, "Tags") return } - for zxvk := range z.Tags { - err = en.WriteString(z.Tags[zxvk]) + for za0001 := range z.Tags { + err = en.WriteString(z.Tags[za0001]) if err != nil { + err = msgp.WrapError(err, "Tags", za0001) return } } @@ -199,9 +208,9 @@ func (z *MetricData) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *MetricData) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 10 + // map header, size 9 // string "Id" - o = append(o, 0x8a, 0xa2, 0x49, 0x64) + o = append(o, 0x89, 0xa2, 0x49, 0x64) o = msgp.AppendString(o, z.Id) // string "OrgId" o = append(o, 0xa5, 0x4f, 0x72, 0x67, 0x49, 0x64) @@ -209,9 +218,6 @@ func (z *MetricData) MarshalMsg(b []byte) (o []byte, err error) { // string "Name" o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) o = msgp.AppendString(o, z.Name) - // string "Metric" - o = append(o, 0xa6, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63) - o = msgp.AppendString(o, z.Metric) // string "Interval" o = append(o, 0xa8, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c) o = msgp.AppendInt(o, z.Interval) @@ -230,8 +236,8 @@ func (z *MetricData) MarshalMsg(b []byte) (o []byte, err error) { // string "Tags" o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73) o = msgp.AppendArrayHeader(o, uint32(len(z.Tags))) - for zxvk := range z.Tags { - o = msgp.AppendString(o, z.Tags[zxvk]) + for za0001 := range z.Tags { + o = msgp.AppendString(o, z.Tags[za0001]) } return } @@ -240,83 +246,91 @@ func (z *MetricData) MarshalMsg(b []byte) (o []byte, err error) { func (z *MetricData) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zcmr uint32 - zcmr, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { + err = msgp.WrapError(err) return } - for zcmr > 0 { - zcmr-- + for zb0001 > 0 { + zb0001-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Id": z.Id, bts, err = msgp.ReadStringBytes(bts) if err != nil { + err = msgp.WrapError(err, "Id") return } case "OrgId": z.OrgId, bts, err = msgp.ReadIntBytes(bts) if err != nil { + err = msgp.WrapError(err, "OrgId") return } case "Name": z.Name, bts, err = msgp.ReadStringBytes(bts) if err != nil { - return - } - case "Metric": - z.Metric, bts, err = msgp.ReadStringBytes(bts) - if err != nil { + err = msgp.WrapError(err, "Name") return } case "Interval": z.Interval, bts, err = msgp.ReadIntBytes(bts) if err != nil { + err = msgp.WrapError(err, "Interval") return } case "Value": z.Value, bts, err = msgp.ReadFloat64Bytes(bts) if err != nil { + err = msgp.WrapError(err, "Value") return } case "Unit": z.Unit, bts, err = msgp.ReadStringBytes(bts) if err != nil { + err = msgp.WrapError(err, "Unit") return } case "Time": z.Time, bts, err = msgp.ReadInt64Bytes(bts) if err != nil { + err = msgp.WrapError(err, "Time") return } case "Mtype": z.Mtype, bts, err = msgp.ReadStringBytes(bts) if err != nil { + err = msgp.WrapError(err, "Mtype") return } case "Tags": - var zajw uint32 - zajw, bts, err = msgp.ReadArrayHeaderBytes(bts) + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { + err = msgp.WrapError(err, "Tags") return } - if cap(z.Tags) >= int(zajw) { - z.Tags = (z.Tags)[:zajw] + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] } else { - z.Tags = make([]string, zajw) + z.Tags = make([]string, zb0002) } - for zxvk := range z.Tags { - z.Tags[zxvk], bts, err = msgp.ReadStringBytes(bts) + for za0001 := range z.Tags { + z.Tags[za0001], bts, err = msgp.ReadStringBytes(bts) if err != nil { + err = msgp.WrapError(err, "Tags", za0001) return } } default: bts, err = msgp.Skip(bts) if err != nil { + err = msgp.WrapError(err) return } } @@ -327,38 +341,41 @@ func (z *MetricData) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *MetricData) Msgsize() (s int) { - s = 1 + 3 + msgp.StringPrefixSize + len(z.Id) + 6 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.StringPrefixSize + len(z.Metric) + 9 + msgp.IntSize + 6 + msgp.Float64Size + 5 + msgp.StringPrefixSize + len(z.Unit) + 5 + msgp.Int64Size + 6 + msgp.StringPrefixSize + len(z.Mtype) + 5 + msgp.ArrayHeaderSize - for zxvk := range z.Tags { - s += msgp.StringPrefixSize + len(z.Tags[zxvk]) + s = 1 + 3 + msgp.StringPrefixSize + len(z.Id) + 6 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Name) + 9 + msgp.IntSize + 6 + msgp.Float64Size + 5 + msgp.StringPrefixSize + len(z.Unit) + 5 + msgp.Int64Size + 6 + msgp.StringPrefixSize + len(z.Mtype) + 5 + msgp.ArrayHeaderSize + for za0001 := range z.Tags { + s += msgp.StringPrefixSize + len(z.Tags[za0001]) } return } // DecodeMsg implements msgp.Decodable func (z *MetricDataArray) DecodeMsg(dc *msgp.Reader) (err error) { - var zcua uint32 - zcua, err = dc.ReadArrayHeader() + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err) return } - if cap((*z)) >= int(zcua) { - (*z) = (*z)[:zcua] + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] } else { - (*z) = make(MetricDataArray, zcua) + (*z) = make(MetricDataArray, zb0002) } - for zhct := range *z { + for zb0001 := range *z { if dc.IsNil() { err = dc.ReadNil() if err != nil { + err = msgp.WrapError(err, zb0001) return } - (*z)[zhct] = nil + (*z)[zb0001] = nil } else { - if (*z)[zhct] == nil { - (*z)[zhct] = new(MetricData) + if (*z)[zb0001] == nil { + (*z)[zb0001] = new(MetricData) } - err = (*z)[zhct].DecodeMsg(dc) + err = (*z)[zb0001].DecodeMsg(dc) if err != nil { + err = msgp.WrapError(err, zb0001) return } } @@ -370,17 +387,19 @@ func (z *MetricDataArray) DecodeMsg(dc *msgp.Reader) (err error) { func (z MetricDataArray) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteArrayHeader(uint32(len(z))) if err != nil { + err = msgp.WrapError(err) return } - for zxhx := range z { - if z[zxhx] == nil { + for zb0003 := range z { + if z[zb0003] == nil { err = en.WriteNil() if err != nil { return } } else { - err = z[zxhx].EncodeMsg(en) + err = z[zb0003].EncodeMsg(en) if err != nil { + err = msgp.WrapError(err, zb0003) return } } @@ -392,12 +411,13 @@ func (z MetricDataArray) EncodeMsg(en *msgp.Writer) (err error) { func (z MetricDataArray) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) o = msgp.AppendArrayHeader(o, uint32(len(z))) - for zxhx := range z { - if z[zxhx] == nil { + for zb0003 := range z { + if z[zb0003] == nil { o = msgp.AppendNil(o) } else { - o, err = z[zxhx].MarshalMsg(o) + o, err = z[zb0003].MarshalMsg(o) if err != nil { + err = msgp.WrapError(err, zb0003) return } } @@ -407,29 +427,31 @@ func (z MetricDataArray) MarshalMsg(b []byte) (o []byte, err error) { // UnmarshalMsg implements msgp.Unmarshaler func (z *MetricDataArray) UnmarshalMsg(bts []byte) (o []byte, err error) { - var zdaf uint32 - zdaf, bts, err = msgp.ReadArrayHeaderBytes(bts) + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { + err = msgp.WrapError(err) return } - if cap((*z)) >= int(zdaf) { - (*z) = (*z)[:zdaf] + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] } else { - (*z) = make(MetricDataArray, zdaf) + (*z) = make(MetricDataArray, zb0002) } - for zlqf := range *z { + for zb0001 := range *z { if msgp.IsNil(bts) { bts, err = msgp.ReadNilBytes(bts) if err != nil { return } - (*z)[zlqf] = nil + (*z)[zb0001] = nil } else { - if (*z)[zlqf] == nil { - (*z)[zlqf] = new(MetricData) + if (*z)[zb0001] == nil { + (*z)[zb0001] = new(MetricData) } - bts, err = (*z)[zlqf].UnmarshalMsg(bts) + bts, err = (*z)[zb0001].UnmarshalMsg(bts) if err != nil { + err = msgp.WrapError(err, zb0001) return } } @@ -441,11 +463,11 @@ func (z *MetricDataArray) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z MetricDataArray) Msgsize() (s int) { s = msgp.ArrayHeaderSize - for zpks := range z { - if z[zpks] == nil { + for zb0003 := range z { + if z[zb0003] == nil { s += msgp.NilSize } else { - s += z[zpks].Msgsize() + s += z[zb0003].Msgsize() } } return @@ -455,83 +477,91 @@ func (z MetricDataArray) Msgsize() (s int) { func (z *MetricDefinition) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field - var zcxo uint32 - zcxo, err = dc.ReadMapHeader() + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err) return } - for zcxo > 0 { - zcxo-- + for zb0001 > 0 { + zb0001-- field, err = dc.ReadMapKeyPtr() if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Id": - z.Id, err = dc.ReadString() + err = z.Id.DecodeMsg(dc) if err != nil { + err = msgp.WrapError(err, "Id") return } case "OrgId": - z.OrgId, err = dc.ReadInt() + z.OrgId, err = dc.ReadUint32() if err != nil { + err = msgp.WrapError(err, "OrgId") return } case "Name": z.Name, err = dc.ReadString() if err != nil { - return - } - case "Metric": - z.Metric, err = dc.ReadString() - if err != nil { + err = msgp.WrapError(err, "Name") return } case "Interval": z.Interval, err = dc.ReadInt() if err != nil { + err = msgp.WrapError(err, "Interval") return } case "Unit": z.Unit, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Unit") return } case "Mtype": z.Mtype, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Mtype") return } case "Tags": - var zeff uint32 - zeff, err = dc.ReadArrayHeader() + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err, "Tags") return } - if cap(z.Tags) >= int(zeff) { - z.Tags = (z.Tags)[:zeff] + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] } else { - z.Tags = make([]string, zeff) + z.Tags = make([]string, zb0002) } - for zjfb := range z.Tags { - z.Tags[zjfb], err = dc.ReadString() + for za0001 := range z.Tags { + z.Tags[za0001], err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Tags", za0001) return } } case "LastUpdate": z.LastUpdate, err = dc.ReadInt64() if err != nil { + err = msgp.WrapError(err, "LastUpdate") return } case "Partition": z.Partition, err = dc.ReadInt32() if err != nil { + err = msgp.WrapError(err, "Partition") return } default: err = dc.Skip() if err != nil { + err = msgp.WrapError(err) return } } @@ -541,101 +571,102 @@ func (z *MetricDefinition) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *MetricDefinition) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 10 + // map header, size 9 // write "Id" - err = en.Append(0x8a, 0xa2, 0x49, 0x64) + err = en.Append(0x89, 0xa2, 0x49, 0x64) if err != nil { - return err + return } - err = en.WriteString(z.Id) + err = z.Id.EncodeMsg(en) if err != nil { + err = msgp.WrapError(err, "Id") return } // write "OrgId" err = en.Append(0xa5, 0x4f, 0x72, 0x67, 0x49, 0x64) if err != nil { - return err + return } - err = en.WriteInt(z.OrgId) + err = en.WriteUint32(z.OrgId) if err != nil { + err = msgp.WrapError(err, "OrgId") return } // write "Name" err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65) - if err != nil { - return err - } - err = en.WriteString(z.Name) if err != nil { return } - // write "Metric" - err = en.Append(0xa6, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63) - if err != nil { - return err - } - err = en.WriteString(z.Metric) + err = en.WriteString(z.Name) if err != nil { + err = msgp.WrapError(err, "Name") return } // write "Interval" err = en.Append(0xa8, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c) if err != nil { - return err + return } err = en.WriteInt(z.Interval) if err != nil { + err = msgp.WrapError(err, "Interval") return } // write "Unit" err = en.Append(0xa4, 0x55, 0x6e, 0x69, 0x74) if err != nil { - return err + return } err = en.WriteString(z.Unit) if err != nil { + err = msgp.WrapError(err, "Unit") return } // write "Mtype" err = en.Append(0xa5, 0x4d, 0x74, 0x79, 0x70, 0x65) if err != nil { - return err + return } err = en.WriteString(z.Mtype) if err != nil { + err = msgp.WrapError(err, "Mtype") return } // write "Tags" err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73) if err != nil { - return err + return } err = en.WriteArrayHeader(uint32(len(z.Tags))) if err != nil { + err = msgp.WrapError(err, "Tags") return } - for zjfb := range z.Tags { - err = en.WriteString(z.Tags[zjfb]) + for za0001 := range z.Tags { + err = en.WriteString(z.Tags[za0001]) if err != nil { + err = msgp.WrapError(err, "Tags", za0001) return } } // write "LastUpdate" err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) if err != nil { - return err + return } err = en.WriteInt64(z.LastUpdate) if err != nil { + err = msgp.WrapError(err, "LastUpdate") return } // write "Partition" err = en.Append(0xa9, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e) if err != nil { - return err + return } err = en.WriteInt32(z.Partition) if err != nil { + err = msgp.WrapError(err, "Partition") return } return @@ -644,19 +675,20 @@ func (z *MetricDefinition) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *MetricDefinition) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 10 + // map header, size 9 // string "Id" - o = append(o, 0x8a, 0xa2, 0x49, 0x64) - o = msgp.AppendString(o, z.Id) + o = append(o, 0x89, 0xa2, 0x49, 0x64) + o, err = z.Id.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Id") + return + } // string "OrgId" o = append(o, 0xa5, 0x4f, 0x72, 0x67, 0x49, 0x64) - o = msgp.AppendInt(o, z.OrgId) + o = msgp.AppendUint32(o, z.OrgId) // string "Name" o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) o = msgp.AppendString(o, z.Name) - // string "Metric" - o = append(o, 0xa6, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63) - o = msgp.AppendString(o, z.Metric) // string "Interval" o = append(o, 0xa8, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c) o = msgp.AppendInt(o, z.Interval) @@ -669,8 +701,8 @@ func (z *MetricDefinition) MarshalMsg(b []byte) (o []byte, err error) { // string "Tags" o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73) o = msgp.AppendArrayHeader(o, uint32(len(z.Tags))) - for zjfb := range z.Tags { - o = msgp.AppendString(o, z.Tags[zjfb]) + for za0001 := range z.Tags { + o = msgp.AppendString(o, z.Tags[za0001]) } // string "LastUpdate" o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) @@ -685,83 +717,91 @@ func (z *MetricDefinition) MarshalMsg(b []byte) (o []byte, err error) { func (z *MetricDefinition) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zrsw uint32 - zrsw, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { + err = msgp.WrapError(err) return } - for zrsw > 0 { - zrsw-- + for zb0001 > 0 { + zb0001-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Id": - z.Id, bts, err = msgp.ReadStringBytes(bts) + bts, err = z.Id.UnmarshalMsg(bts) if err != nil { + err = msgp.WrapError(err, "Id") return } case "OrgId": - z.OrgId, bts, err = msgp.ReadIntBytes(bts) + z.OrgId, bts, err = msgp.ReadUint32Bytes(bts) if err != nil { + err = msgp.WrapError(err, "OrgId") return } case "Name": z.Name, bts, err = msgp.ReadStringBytes(bts) if err != nil { - return - } - case "Metric": - z.Metric, bts, err = msgp.ReadStringBytes(bts) - if err != nil { + err = msgp.WrapError(err, "Name") return } case "Interval": z.Interval, bts, err = msgp.ReadIntBytes(bts) if err != nil { + err = msgp.WrapError(err, "Interval") return } case "Unit": z.Unit, bts, err = msgp.ReadStringBytes(bts) if err != nil { + err = msgp.WrapError(err, "Unit") return } case "Mtype": z.Mtype, bts, err = msgp.ReadStringBytes(bts) if err != nil { + err = msgp.WrapError(err, "Mtype") return } case "Tags": - var zxpk uint32 - zxpk, bts, err = msgp.ReadArrayHeaderBytes(bts) + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { + err = msgp.WrapError(err, "Tags") return } - if cap(z.Tags) >= int(zxpk) { - z.Tags = (z.Tags)[:zxpk] + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] } else { - z.Tags = make([]string, zxpk) + z.Tags = make([]string, zb0002) } - for zjfb := range z.Tags { - z.Tags[zjfb], bts, err = msgp.ReadStringBytes(bts) + for za0001 := range z.Tags { + z.Tags[za0001], bts, err = msgp.ReadStringBytes(bts) if err != nil { + err = msgp.WrapError(err, "Tags", za0001) return } } case "LastUpdate": z.LastUpdate, bts, err = msgp.ReadInt64Bytes(bts) if err != nil { + err = msgp.WrapError(err, "LastUpdate") return } case "Partition": z.Partition, bts, err = msgp.ReadInt32Bytes(bts) if err != nil { + err = msgp.WrapError(err, "Partition") return } default: bts, err = msgp.Skip(bts) if err != nil { + err = msgp.WrapError(err) return } } @@ -772,9 +812,9 @@ func (z *MetricDefinition) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *MetricDefinition) Msgsize() (s int) { - s = 1 + 3 + msgp.StringPrefixSize + len(z.Id) + 6 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.StringPrefixSize + len(z.Metric) + 9 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Unit) + 6 + msgp.StringPrefixSize + len(z.Mtype) + 5 + msgp.ArrayHeaderSize - for zjfb := range z.Tags { - s += msgp.StringPrefixSize + len(z.Tags[zjfb]) + s = 1 + 3 + z.Id.Msgsize() + 6 + msgp.Uint32Size + 5 + msgp.StringPrefixSize + len(z.Name) + 9 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Unit) + 6 + msgp.StringPrefixSize + len(z.Mtype) + 5 + msgp.ArrayHeaderSize + for za0001 := range z.Tags { + s += msgp.StringPrefixSize + len(z.Tags[za0001]) } s += 11 + msgp.Int64Size + 10 + msgp.Int32Size return diff --git a/vendor/github.com/grafana/metrictank/schema/metricpoint.go b/vendor/github.com/grafana/metrictank/schema/metricpoint.go new file mode 100644 index 00000000..9461b634 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/metricpoint.go @@ -0,0 +1,93 @@ +package schema + +import ( + "encoding/binary" + "math" +) + +//msgp:ignore MetricPoint + +// MetricPoint is a simple way to represent a point of a metric +// it can be serialized/deserialized with and without the Org field +// of the key. +type MetricPoint struct { + MKey MKey + Value float64 + Time uint32 +} + +func (m *MetricPoint) Valid() bool { + return m.Time != 0 +} + +// MarshalWithoutOrg28 marshals directly to b in a non-multi-tenancy aware way +// b must have a cap-len difference of at least 28 bytes. +func (m *MetricPoint) MarshalWithoutOrg28(b []byte) (o []byte, err error) { + l := len(b) + b = b[:l+28] + m.putWithoutOrg(b[l:]) + return b, nil +} + +// MarshalWithoutOrg marshals in a non-multi-tenancy aware way +func (m *MetricPoint) MarshalWithoutOrg(b []byte) (o []byte, err error) { + l := len(b) + b = ensure(b, 28) // 16+8+4 + m.putWithoutOrg(b[l:]) + return b, nil +} + +func (m *MetricPoint) putWithoutOrg(b []byte) { + copy(b, m.MKey.Key[:]) + binary.LittleEndian.PutUint64(b[16:], math.Float64bits(m.Value)) + binary.LittleEndian.PutUint32(b[24:], m.Time) +} + +// UnmarshalWithoutOrg unmarshals in a non-multi-tenancy aware way +func (m *MetricPoint) UnmarshalWithoutOrg(bts []byte) (o []byte, err error) { + copy(m.MKey.Key[:], bts[:16]) + m.Value = math.Float64frombits(binary.LittleEndian.Uint64(bts[16:24])) + m.Time = binary.LittleEndian.Uint32(bts[24:]) + return bts[28:], nil +} + +// Marshal32 marshals the MetricPoint directly to b. +// b must have a cap-len difference of at least 32 bytes. +func (m *MetricPoint) Marshal32(b []byte) (o []byte, err error) { + l := len(b) + b = b[:l+32] + m.putWithoutOrg(b[l:]) + binary.LittleEndian.PutUint32(b[l+28:], m.MKey.Org) + return b, nil +} + +// Marshal marshals the MetricPoint +func (m *MetricPoint) Marshal(b []byte) (o []byte, err error) { + l := len(b) + b = ensure(b, 32) // 16+8+4+4 + m.putWithoutOrg(b[l:]) + binary.LittleEndian.PutUint32(b[l+28:], m.MKey.Org) + return b, nil +} + +// Unmarshal unmarshals the MetricPoint +func (m *MetricPoint) Unmarshal(bts []byte) (o []byte, err error) { + copy(m.MKey.Key[:], bts[:16]) + m.Value = math.Float64frombits(binary.LittleEndian.Uint64(bts[16:24])) + m.Time = binary.LittleEndian.Uint32(bts[24:]) + m.MKey.Org = binary.LittleEndian.Uint32(bts[28:]) + return bts[32:], nil +} + +// ensure 'sz' extra bytes in 'b' btw len(b) and cap(b) +// and returns a slice with updated length +func ensure(b []byte, sz int) []byte { + l := len(b) + c := cap(b) + if c-l < sz { + o := make([]byte, (2*c)+sz) // exponential growth + n := copy(o, b) + return o[:n+sz] + } + return b[:l+sz] +} diff --git a/vendor/gopkg.in/raintank/schema.v1/msg/format.go b/vendor/github.com/grafana/metrictank/schema/msg/format.go similarity index 72% rename from vendor/gopkg.in/raintank/schema.v1/msg/format.go rename to vendor/github.com/grafana/metrictank/schema/msg/format.go index 59803fdc..d813ca14 100644 --- a/vendor/gopkg.in/raintank/schema.v1/msg/format.go +++ b/vendor/github.com/grafana/metrictank/schema/msg/format.go @@ -8,9 +8,6 @@ type Format uint8 const ( FormatMetricDataArrayJson Format = iota FormatMetricDataArrayMsgp -) - -const ( - FormatProbeEventJson Format = iota - FormatProbeEventMsgp + FormatMetricPoint + FormatMetricPointWithoutOrg ) diff --git a/vendor/github.com/grafana/metrictank/schema/msg/format_string.go b/vendor/github.com/grafana/metrictank/schema/msg/format_string.go new file mode 100644 index 00000000..a37a9109 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/msg/format_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=Format"; DO NOT EDIT. + +package msg + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[FormatMetricDataArrayJson-0] + _ = x[FormatMetricDataArrayMsgp-1] + _ = x[FormatMetricPoint-2] + _ = x[FormatMetricPointWithoutOrg-3] +} + +const _Format_name = "FormatMetricDataArrayJsonFormatMetricDataArrayMsgpFormatMetricPointFormatMetricPointWithoutOrg" + +var _Format_index = [...]uint8{0, 25, 50, 67, 94} + +func (i Format) String() string { + if i >= Format(len(_Format_index)-1) { + return "Format(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Format_name[_Format_index[i]:_Format_index[i+1]] +} diff --git a/vendor/github.com/grafana/metrictank/schema/msg/msg.go b/vendor/github.com/grafana/metrictank/schema/msg/msg.go new file mode 100644 index 00000000..79077ac6 --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/msg/msg.go @@ -0,0 +1,142 @@ +package msg + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/grafana/metrictank/schema" +) + +var errTooSmall = errors.New("too small") +var errFmtBinWriteFailed = "binary write failed: %q" +var errFmtUnsupportedFormat = "unsupported format %d" + +type MetricData struct { + Id int64 + Metrics []*schema.MetricData + Produced time.Time + Format Format + Msg []byte +} + +// parses format and id (cheap), but doesn't decode metrics (expensive) just yet. +func (m *MetricData) InitFromMsg(msg []byte) error { + if len(msg) < 9 { + return errTooSmall + } + m.Msg = msg + + buf := bytes.NewReader(msg[1:9]) + binary.Read(buf, binary.BigEndian, &m.Id) + m.Produced = time.Unix(0, m.Id) + + m.Format = Format(msg[0]) + if m.Format != FormatMetricDataArrayJson && m.Format != FormatMetricDataArrayMsgp { + return fmt.Errorf(errFmtUnsupportedFormat, m.Format) + } + return nil +} + +// sets m.Metrics to a []*schema.MetricData +// any subsequent call may however put different MetricData into our m.Metrics array +func (m *MetricData) DecodeMetricData() error { + var err error + switch m.Format { + case FormatMetricDataArrayJson: + err = json.Unmarshal(m.Msg[9:], &m.Metrics) + case FormatMetricDataArrayMsgp: + out := schema.MetricDataArray(m.Metrics) + _, err = out.UnmarshalMsg(m.Msg[9:]) + m.Metrics = []*schema.MetricData(out) + default: + return fmt.Errorf("unrecognized format %d", m.Msg[0]) + } + if err != nil { + return fmt.Errorf("ERROR: failure to unmarshal message body via format %q: %s", m.Format, err) + } + m.Msg = nil // no more need for the original input + return nil +} + +// CreateMsg is the legacy function to create messages. It's not very fast +func CreateMsg(metrics []*schema.MetricData, id int64, version Format) ([]byte, error) { + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, uint8(version)) + if err != nil { + return nil, fmt.Errorf(errFmtBinWriteFailed, err) + } + err = binary.Write(buf, binary.BigEndian, id) + if err != nil { + return nil, fmt.Errorf(errFmtBinWriteFailed, err) + } + var msg []byte + switch version { + case FormatMetricDataArrayJson: + msg, err = json.Marshal(metrics) + case FormatMetricDataArrayMsgp: + m := schema.MetricDataArray(metrics) + msg, err = m.MarshalMsg(nil) + default: + return nil, fmt.Errorf(errFmtUnsupportedFormat, version) + } + if err != nil { + return nil, fmt.Errorf("Failed to marshal metrics payload: %s", err) + } + _, err = buf.Write(msg) + if err != nil { + return nil, fmt.Errorf(errFmtBinWriteFailed, err) + } + return buf.Bytes(), nil +} + +// WritePointMsg is like CreateMsg, except optimized for MetricPoint and buffer re-use. +// caller must assure a cap-len diff of at least: +// 33B (for FormatMetricPoint) +// 29B (for FormatMetricPointWithoutOrg) +// no other formats supported. +func WritePointMsg(point schema.MetricPoint, buf []byte, version Format) (o []byte, err error) { + b := buf[:1] + switch version { + case FormatMetricPoint: + b[0] = byte(FormatMetricPoint) + return point.Marshal32(b) + case FormatMetricPointWithoutOrg: + b[0] = byte(FormatMetricPointWithoutOrg) + return point.MarshalWithoutOrg28(b) + } + return nil, fmt.Errorf(errFmtUnsupportedFormat, version) +} + +func IsPointMsg(data []byte) (Format, bool) { + l := len(data) + if l == 0 { + return 0, false + } + version := Format(data[0]) + if l == 29 && version == FormatMetricPointWithoutOrg { + return FormatMetricPointWithoutOrg, true + } + if l == 33 && version == FormatMetricPoint { + return FormatMetricPoint, true + } + return 0, false +} + +func ReadPointMsg(data []byte, defaultOrg uint32) ([]byte, schema.MetricPoint, error) { + var point schema.MetricPoint + version := Format(data[0]) + if len(data) == 29 && version == FormatMetricPointWithoutOrg { + o, err := point.UnmarshalWithoutOrg(data[1:]) + point.MKey.Org = defaultOrg + return o, point, err + } + if len(data) == 33 && version == FormatMetricPoint { + o, err := point.Unmarshal(data[1:]) + return o, point, err + } + return data, point, fmt.Errorf(errFmtUnsupportedFormat, version) +} diff --git a/vendor/github.com/grafana/metrictank/schema/partition.go b/vendor/github.com/grafana/metrictank/schema/partition.go new file mode 100644 index 00000000..f1064c2f --- /dev/null +++ b/vendor/github.com/grafana/metrictank/schema/partition.go @@ -0,0 +1,116 @@ +package schema + +import ( + "encoding/binary" + "hash/fnv" + + "github.com/cespare/xxhash" + jump "github.com/dgryski/go-jump" +) + +type PartitionByMethod uint8 + +const ( + // partition by organization id only + PartitionByOrg PartitionByMethod = iota + + // partition by the metric name only + PartitionBySeries + + // partition by metric name and tags, with the best distribution + // recommended for new deployments. + PartitionBySeriesWithTags + + // partition by metric name and tags, with a sub-optimal distribution when using tags. + // compatible with PartitionBySeries if a metric has no tags, + // making it possible to adopt tags for existing PartitionBySeries deployments without a migration. + PartitionBySeriesWithTagsFnv +) + +func (m *MetricData) PartitionID(method PartitionByMethod, partitions int32) (int32, error) { + var partition int32 + + switch method { + case PartitionByOrg: + h := fnv.New32a() + err := binary.Write(h, binary.LittleEndian, uint32(m.OrgId)) + if err != nil { + return 0, err + } + partition = int32(h.Sum32()) % partitions + if partition < 0 { + partition = -partition + } + case PartitionBySeries: + h := fnv.New32a() + h.Write([]byte(m.Name)) + partition = int32(h.Sum32()) % partitions + if partition < 0 { + partition = -partition + } + case PartitionBySeriesWithTags: + h := xxhash.New() + if err := writeSortedTagString(h, m.Name, m.Tags); err != nil { + return 0, err + } + partition = jump.Hash(h.Sum64(), int(partitions)) + case PartitionBySeriesWithTagsFnv: + h := fnv.New32a() + if err := writeSortedTagString(h, m.Name, m.Tags); err != nil { + return 0, err + } + partition = int32(h.Sum32()) % partitions + if partition < 0 { + partition = -partition + } + default: + return 0, ErrUnknownPartitionMethod + } + + return partition, nil +} + +func (m *MetricDefinition) PartitionID(method PartitionByMethod, partitions int32) (int32, error) { + var partition int32 + + switch method { + case PartitionByOrg: + h := fnv.New32a() + err := binary.Write(h, binary.LittleEndian, uint32(m.OrgId)) + if err != nil { + return 0, err + } + partition = int32(h.Sum32()) % partitions + if partition < 0 { + partition = -partition + } + case PartitionBySeries: + h := fnv.New32a() + h.Write([]byte(m.Name)) + partition = int32(h.Sum32()) % partitions + if partition < 0 { + partition = -partition + } + case PartitionBySeriesWithTags: + h := xxhash.New() + h.WriteString(m.NameWithTags()) + partition = jump.Hash(h.Sum64(), int(partitions)) + case PartitionBySeriesWithTagsFnv: + h := fnv.New32a() + if len(m.nameWithTags) > 0 { + h.Write([]byte(m.nameWithTags)) + } else { + if err := writeSortedTagString(h, m.Name, m.Tags); err != nil { + return 0, err + } + } + partition = int32(h.Sum32()) % partitions + if partition < 0 { + partition = -partition + } + default: + return 0, ErrUnknownPartitionMethod + } + + return partition, nil +} diff --git a/vendor/gopkg.in/raintank/schema.v1/point.go b/vendor/github.com/grafana/metrictank/schema/point.go similarity index 100% rename from vendor/gopkg.in/raintank/schema.v1/point.go rename to vendor/github.com/grafana/metrictank/schema/point.go diff --git a/vendor/gopkg.in/raintank/schema.v1/point_gen.go b/vendor/github.com/grafana/metrictank/schema/point_gen.go similarity index 76% rename from vendor/gopkg.in/raintank/schema.v1/point_gen.go rename to vendor/github.com/grafana/metrictank/schema/point_gen.go index 6843d766..f82bb3cf 100644 --- a/vendor/gopkg.in/raintank/schema.v1/point_gen.go +++ b/vendor/github.com/grafana/metrictank/schema/point_gen.go @@ -1,8 +1,6 @@ package schema -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT +// Code generated by github.com/tinylib/msgp DO NOT EDIT. import ( "github.com/tinylib/msgp/msgp" @@ -12,31 +10,36 @@ import ( func (z *Point) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field - var zxvk uint32 - zxvk, err = dc.ReadMapHeader() + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err) return } - for zxvk > 0 { - zxvk-- + for zb0001 > 0 { + zb0001-- field, err = dc.ReadMapKeyPtr() if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Val": z.Val, err = dc.ReadFloat64() if err != nil { + err = msgp.WrapError(err, "Val") return } case "Ts": z.Ts, err = dc.ReadUint32() if err != nil { + err = msgp.WrapError(err, "Ts") return } default: err = dc.Skip() if err != nil { + err = msgp.WrapError(err) return } } @@ -50,19 +53,21 @@ func (z Point) EncodeMsg(en *msgp.Writer) (err error) { // write "Val" err = en.Append(0x82, 0xa3, 0x56, 0x61, 0x6c) if err != nil { - return err + return } err = en.WriteFloat64(z.Val) if err != nil { + err = msgp.WrapError(err, "Val") return } // write "Ts" err = en.Append(0xa2, 0x54, 0x73) if err != nil { - return err + return } err = en.WriteUint32(z.Ts) if err != nil { + err = msgp.WrapError(err, "Ts") return } return @@ -85,31 +90,36 @@ func (z Point) MarshalMsg(b []byte) (o []byte, err error) { func (z *Point) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zbzg uint32 - zbzg, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { + err = msgp.WrapError(err) return } - for zbzg > 0 { - zbzg-- + for zb0001 > 0 { + zb0001-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Val": z.Val, bts, err = msgp.ReadFloat64Bytes(bts) if err != nil { + err = msgp.WrapError(err, "Val") return } case "Ts": z.Ts, bts, err = msgp.ReadUint32Bytes(bts) if err != nil { + err = msgp.WrapError(err, "Ts") return } default: bts, err = msgp.Skip(bts) if err != nil { + err = msgp.WrapError(err) return } } diff --git a/vendor/gopkg.in/raintank/schema.v1/reslice.go b/vendor/github.com/grafana/metrictank/schema/reslice.go similarity index 100% rename from vendor/gopkg.in/raintank/schema.v1/reslice.go rename to vendor/github.com/grafana/metrictank/schema/reslice.go diff --git a/vendor/github.com/raintank/metrictank/cluster/partitioner/partitioner.go b/vendor/github.com/raintank/metrictank/cluster/partitioner/partitioner.go deleted file mode 100644 index 75a2329d..00000000 --- a/vendor/github.com/raintank/metrictank/cluster/partitioner/partitioner.go +++ /dev/null @@ -1,52 +0,0 @@ -package partitioner - -import ( - "fmt" - - "github.com/Shopify/sarama" - "gopkg.in/raintank/schema.v1" -) - -type Partitioner interface { - Partition(schema.PartitionedMetric, int32) (int32, error) -} - -type Kafka struct { - PartitionBy string - Partitioner sarama.Partitioner -} - -func NewKafka(partitionBy string) (*Kafka, error) { - switch partitionBy { - case "byOrg": - case "bySeries": - default: - return nil, fmt.Errorf("partitionBy must be one of 'byOrg|bySeries'. got %s", partitionBy) - } - return &Kafka{ - PartitionBy: partitionBy, - Partitioner: sarama.NewHashPartitioner(""), - }, nil -} - -func (k *Kafka) Partition(m schema.PartitionedMetric, numPartitions int32) (int32, error) { - key, err := k.GetPartitionKey(m, nil) - if err != nil { - return 0, err - } - return k.Partitioner.Partition(&sarama.ProducerMessage{Key: sarama.ByteEncoder(key)}, numPartitions) -} - -func (k *Kafka) GetPartitionKey(m schema.PartitionedMetric, b []byte) ([]byte, error) { - switch k.PartitionBy { - case "byOrg": - // partition by organisation: metrics for the same org should go to the same - // partition/MetricTank (optimize for locality~performance) - return m.KeyByOrgId(b), nil - case "bySeries": - // partition by series: metrics are distrubted across all metrictank instances - // to allow horizontal scalability - return m.KeyBySeries(b), nil - } - return b, fmt.Errorf("unkown partitionBy setting.") -} diff --git a/vendor/github.com/tinylib/msgp/msgp/edit.go b/vendor/github.com/tinylib/msgp/msgp/edit.go index 41f92986..b473a6f6 100644 --- a/vendor/github.com/tinylib/msgp/msgp/edit.go +++ b/vendor/github.com/tinylib/msgp/msgp/edit.go @@ -198,6 +198,7 @@ func resizeMap(raw []byte, delta int64) []byte { if cap(raw)-len(raw) >= 2 { raw = raw[0 : len(raw)+2] copy(raw[5:], raw[3:]) + raw[0] = mmap32 big.PutUint32(raw[1:], uint32(sz+delta)) return raw } diff --git a/vendor/github.com/tinylib/msgp/msgp/errors.go b/vendor/github.com/tinylib/msgp/msgp/errors.go index 5c24f271..cc78a980 100644 --- a/vendor/github.com/tinylib/msgp/msgp/errors.go +++ b/vendor/github.com/tinylib/msgp/msgp/errors.go @@ -5,6 +5,8 @@ import ( "reflect" ) +const resumableDefault = false + var ( // ErrShortBytes is returned when the // slice being decoded is too short to @@ -26,84 +28,240 @@ type Error interface { // Resumable returns whether // or not the error means that // the stream of data is malformed - // and the information is unrecoverable. + // and the information is unrecoverable. Resumable() bool } +// contextError allows msgp Error instances to be enhanced with additional +// context about their origin. +type contextError interface { + Error + + // withContext must not modify the error instance - it must clone and + // return a new error with the context added. + withContext(ctx string) error +} + +// Cause returns the underlying cause of an error that has been wrapped +// with additional context. +func Cause(e error) error { + out := e + if e, ok := e.(errWrapped); ok && e.cause != nil { + out = e.cause + } + return out +} + +// Resumable returns whether or not the error means that the stream of data is +// malformed and the information is unrecoverable. +func Resumable(e error) bool { + if e, ok := e.(Error); ok { + return e.Resumable() + } + return resumableDefault +} + +// WrapError wraps an error with additional context that allows the part of the +// serialized type that caused the problem to be identified. Underlying errors +// can be retrieved using Cause() +// +// The input error is not modified - a new error should be returned. +// +// ErrShortBytes is not wrapped with any context due to backward compatibility +// issues with the public API. +// +func WrapError(err error, ctx ...interface{}) error { + switch e := err.(type) { + case errShort: + return e + case contextError: + return e.withContext(ctxString(ctx)) + default: + return errWrapped{cause: err, ctx: ctxString(ctx)} + } +} + +// ctxString converts the incoming interface{} slice into a single string. +func ctxString(ctx []interface{}) string { + out := "" + for idx, cv := range ctx { + if idx > 0 { + out += "/" + } + out += fmt.Sprintf("%v", cv) + } + return out +} + +func addCtx(ctx, add string) string { + if ctx != "" { + return add + "/" + ctx + } else { + return add + } +} + +// errWrapped allows arbitrary errors passed to WrapError to be enhanced with +// context and unwrapped with Cause() +type errWrapped struct { + cause error + ctx string +} + +func (e errWrapped) Error() string { + if e.ctx != "" { + return fmt.Sprintf("%s at %s", e.cause, e.ctx) + } else { + return e.cause.Error() + } +} + +func (e errWrapped) Resumable() bool { + if e, ok := e.cause.(Error); ok { + return e.Resumable() + } + return resumableDefault +} + type errShort struct{} func (e errShort) Error() string { return "msgp: too few bytes left to read object" } func (e errShort) Resumable() bool { return false } -type errFatal struct{} +type errFatal struct { + ctx string +} + +func (f errFatal) Error() string { + out := "msgp: fatal decoding error (unreachable code)" + if f.ctx != "" { + out += " at " + f.ctx + } + return out +} -func (f errFatal) Error() string { return "msgp: fatal decoding error (unreachable code)" } func (f errFatal) Resumable() bool { return false } +func (f errFatal) withContext(ctx string) error { f.ctx = addCtx(f.ctx, ctx); return f } + // ArrayError is an error returned // when decoding a fix-sized array // of the wrong size type ArrayError struct { Wanted uint32 Got uint32 + ctx string } // Error implements the error interface func (a ArrayError) Error() string { - return fmt.Sprintf("msgp: wanted array of size %d; got %d", a.Wanted, a.Got) + out := fmt.Sprintf("msgp: wanted array of size %d; got %d", a.Wanted, a.Got) + if a.ctx != "" { + out += " at " + a.ctx + } + return out } // Resumable is always 'true' for ArrayErrors func (a ArrayError) Resumable() bool { return true } +func (a ArrayError) withContext(ctx string) error { a.ctx = addCtx(a.ctx, ctx); return a } + // IntOverflow is returned when a call // would downcast an integer to a type // with too few bits to hold its value. type IntOverflow struct { Value int64 // the value of the integer FailedBitsize int // the bit size that the int64 could not fit into + ctx string } // Error implements the error interface func (i IntOverflow) Error() string { - return fmt.Sprintf("msgp: %d overflows int%d", i.Value, i.FailedBitsize) + str := fmt.Sprintf("msgp: %d overflows int%d", i.Value, i.FailedBitsize) + if i.ctx != "" { + str += " at " + i.ctx + } + return str } // Resumable is always 'true' for overflows func (i IntOverflow) Resumable() bool { return true } +func (i IntOverflow) withContext(ctx string) error { i.ctx = addCtx(i.ctx, ctx); return i } + // UintOverflow is returned when a call // would downcast an unsigned integer to a type // with too few bits to hold its value type UintOverflow struct { Value uint64 // value of the uint FailedBitsize int // the bit size that couldn't fit the value + ctx string } // Error implements the error interface func (u UintOverflow) Error() string { - return fmt.Sprintf("msgp: %d overflows uint%d", u.Value, u.FailedBitsize) + str := fmt.Sprintf("msgp: %d overflows uint%d", u.Value, u.FailedBitsize) + if u.ctx != "" { + str += " at " + u.ctx + } + return str } // Resumable is always 'true' for overflows func (u UintOverflow) Resumable() bool { return true } +func (u UintOverflow) withContext(ctx string) error { u.ctx = addCtx(u.ctx, ctx); return u } + +// UintBelowZero is returned when a call +// would cast a signed integer below zero +// to an unsigned integer. +type UintBelowZero struct { + Value int64 // value of the incoming int + ctx string +} + +// Error implements the error interface +func (u UintBelowZero) Error() string { + str := fmt.Sprintf("msgp: attempted to cast int %d to unsigned", u.Value) + if u.ctx != "" { + str += " at " + u.ctx + } + return str +} + +// Resumable is always 'true' for overflows +func (u UintBelowZero) Resumable() bool { return true } + +func (u UintBelowZero) withContext(ctx string) error { + u.ctx = ctx + return u +} + // A TypeError is returned when a particular // decoding method is unsuitable for decoding // a particular MessagePack value. type TypeError struct { Method Type // Type expected by method Encoded Type // Type actually encoded + + ctx string } // Error implements the error interface func (t TypeError) Error() string { - return fmt.Sprintf("msgp: attempted to decode type %q with method for %q", t.Encoded, t.Method) + out := fmt.Sprintf("msgp: attempted to decode type %q with method for %q", t.Encoded, t.Method) + if t.ctx != "" { + out += " at " + t.ctx + } + return out } // Resumable returns 'true' for TypeErrors func (t TypeError) Resumable() bool { return true } +func (t TypeError) withContext(ctx string) error { t.ctx = addCtx(t.ctx, ctx); return t } + // returns either InvalidPrefixError or // TypeError depending on whether or not // the prefix is recognized @@ -133,10 +291,24 @@ func (i InvalidPrefixError) Resumable() bool { return false } // to a function that takes `interface{}`. type ErrUnsupportedType struct { T reflect.Type + + ctx string } // Error implements error -func (e *ErrUnsupportedType) Error() string { return fmt.Sprintf("msgp: type %q not supported", e.T) } +func (e *ErrUnsupportedType) Error() string { + out := fmt.Sprintf("msgp: type %q not supported", e.T) + if e.ctx != "" { + out += " at " + e.ctx + } + return out +} // Resumable returns 'true' for ErrUnsupportedType func (e *ErrUnsupportedType) Resumable() bool { return true } + +func (e *ErrUnsupportedType) withContext(ctx string) error { + o := *e + o.ctx = addCtx(o.ctx, ctx) + return &o +} diff --git a/vendor/github.com/tinylib/msgp/msgp/extension.go b/vendor/github.com/tinylib/msgp/msgp/extension.go index 588b18f9..0b31dcdb 100644 --- a/vendor/github.com/tinylib/msgp/msgp/extension.go +++ b/vendor/github.com/tinylib/msgp/msgp/extension.go @@ -445,26 +445,27 @@ func AppendExtension(b []byte, e Extension) ([]byte, error) { o[n] = mfixext16 o[n+1] = byte(e.ExtensionType()) n += 2 - } - switch { - case l < math.MaxUint8: - o, n = ensure(b, l+3) - o[n] = mext8 - o[n+1] = byte(uint8(l)) - o[n+2] = byte(e.ExtensionType()) - n += 3 - case l < math.MaxUint16: - o, n = ensure(b, l+4) - o[n] = mext16 - big.PutUint16(o[n+1:], uint16(l)) - o[n+3] = byte(e.ExtensionType()) - n += 4 default: - o, n = ensure(b, l+6) - o[n] = mext32 - big.PutUint32(o[n+1:], uint32(l)) - o[n+5] = byte(e.ExtensionType()) - n += 6 + switch { + case l < math.MaxUint8: + o, n = ensure(b, l+3) + o[n] = mext8 + o[n+1] = byte(uint8(l)) + o[n+2] = byte(e.ExtensionType()) + n += 3 + case l < math.MaxUint16: + o, n = ensure(b, l+4) + o[n] = mext16 + big.PutUint16(o[n+1:], uint16(l)) + o[n+3] = byte(e.ExtensionType()) + n += 4 + default: + o, n = ensure(b, l+6) + o[n] = mext32 + big.PutUint32(o[n+1:], uint32(l)) + o[n+5] = byte(e.ExtensionType()) + n += 6 + } } return o, e.MarshalBinaryTo(o[n:]) } diff --git a/vendor/github.com/tinylib/msgp/msgp/file.go b/vendor/github.com/tinylib/msgp/msgp/file.go index cbbf56cd..8e7370eb 100644 --- a/vendor/github.com/tinylib/msgp/msgp/file.go +++ b/vendor/github.com/tinylib/msgp/msgp/file.go @@ -1,4 +1,5 @@ -// +build linux,!appengine darwin dragonfly freebsd netbsd openbsd +// +build linux darwin dragonfly freebsd netbsd openbsd +// +build !appengine package msgp diff --git a/vendor/github.com/tinylib/msgp/msgp/appengine.go b/vendor/github.com/tinylib/msgp/msgp/purego.go similarity index 88% rename from vendor/github.com/tinylib/msgp/msgp/appengine.go rename to vendor/github.com/tinylib/msgp/msgp/purego.go index bff9e768..c828f7ec 100644 --- a/vendor/github.com/tinylib/msgp/msgp/appengine.go +++ b/vendor/github.com/tinylib/msgp/msgp/purego.go @@ -1,4 +1,4 @@ -// +build appengine +// +build purego appengine package msgp diff --git a/vendor/github.com/tinylib/msgp/msgp/read.go b/vendor/github.com/tinylib/msgp/msgp/read.go index 52c8c8c2..aa668c57 100644 --- a/vendor/github.com/tinylib/msgp/msgp/read.go +++ b/vendor/github.com/tinylib/msgp/msgp/read.go @@ -146,6 +146,56 @@ func (m *Reader) Read(p []byte) (int, error) { return m.R.Read(p) } +// CopyNext reads the next object from m without decoding it and writes it to w. +// It avoids unnecessary copies internally. +func (m *Reader) CopyNext(w io.Writer) (int64, error) { + sz, o, err := getNextSize(m.R) + if err != nil { + return 0, err + } + + var n int64 + // Opportunistic optimization: if we can fit the whole thing in the m.R + // buffer, then just get a pointer to that, and pass it to w.Write, + // avoiding an allocation. + if int(sz) <= m.R.BufferSize() { + var nn int + var buf []byte + buf, err = m.R.Next(int(sz)) + if err != nil { + if err == io.ErrUnexpectedEOF { + err = ErrShortBytes + } + return 0, err + } + nn, err = w.Write(buf) + n += int64(nn) + } else { + // Fall back to io.CopyN. + // May avoid allocating if w is a ReaderFrom (e.g. bytes.Buffer) + n, err = io.CopyN(w, m.R, int64(sz)) + if err == io.ErrUnexpectedEOF { + err = ErrShortBytes + } + } + if err != nil { + return n, err + } else if n < int64(sz) { + return n, io.ErrShortWrite + } + + // for maps and slices, read elements + for x := uintptr(0); x < o; x++ { + var n2 int64 + n2, err = m.CopyNext(w) + if err != nil { + return n, err + } + n += n2 + } + return n, nil +} + // ReadFull implements `io.ReadFull` func (m *Reader) ReadFull(p []byte) (int, error) { return m.R.ReadFull(p) @@ -194,8 +244,10 @@ func (m *Reader) IsNil() bool { return err == nil && p[0] == mnil } +// getNextSize returns the size of the next object on the wire. // returns (obj size, obj elements, error) // only maps and arrays have non-zero obj elements +// for maps and arrays, obj size does not include elements // // use uintptr b/c it's guaranteed to be large enough // to hold whatever we can fit in memory. @@ -531,6 +583,14 @@ func (m *Reader) ReadInt64() (i int64, err error) { i = int64(getMint8(p)) return + case muint8: + p, err = m.R.Next(2) + if err != nil { + return + } + i = int64(getMuint8(p)) + return + case mint16: p, err = m.R.Next(3) if err != nil { @@ -539,6 +599,14 @@ func (m *Reader) ReadInt64() (i int64, err error) { i = int64(getMint16(p)) return + case muint16: + p, err = m.R.Next(3) + if err != nil { + return + } + i = int64(getMuint16(p)) + return + case mint32: p, err = m.R.Next(5) if err != nil { @@ -547,6 +615,14 @@ func (m *Reader) ReadInt64() (i int64, err error) { i = int64(getMint32(p)) return + case muint32: + p, err = m.R.Next(5) + if err != nil { + return + } + i = int64(getMuint32(p)) + return + case mint64: p, err = m.R.Next(9) if err != nil { @@ -555,6 +631,19 @@ func (m *Reader) ReadInt64() (i int64, err error) { i = getMint64(p) return + case muint64: + p, err = m.R.Next(9) + if err != nil { + return + } + u := getMuint64(p) + if u > math.MaxInt64 { + err = UintOverflow{Value: u, FailedBitsize: 64} + return + } + i = int64(u) + return + default: err = badPrefix(IntType, lead) return @@ -626,6 +715,19 @@ func (m *Reader) ReadUint64() (u uint64, err error) { return } switch lead { + case mint8: + p, err = m.R.Next(2) + if err != nil { + return + } + v := int64(getMint8(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + case muint8: p, err = m.R.Next(2) if err != nil { @@ -634,6 +736,19 @@ func (m *Reader) ReadUint64() (u uint64, err error) { u = uint64(getMuint8(p)) return + case mint16: + p, err = m.R.Next(3) + if err != nil { + return + } + v := int64(getMint16(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + case muint16: p, err = m.R.Next(3) if err != nil { @@ -642,6 +757,19 @@ func (m *Reader) ReadUint64() (u uint64, err error) { u = uint64(getMuint16(p)) return + case mint32: + p, err = m.R.Next(5) + if err != nil { + return + } + v := int64(getMint32(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + case muint32: p, err = m.R.Next(5) if err != nil { @@ -650,6 +778,19 @@ func (m *Reader) ReadUint64() (u uint64, err error) { u = uint64(getMuint32(p)) return + case mint64: + p, err = m.R.Next(9) + if err != nil { + return + } + v := int64(getMint64(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + case muint64: p, err = m.R.Next(9) if err != nil { @@ -659,7 +800,11 @@ func (m *Reader) ReadUint64() (u uint64, err error) { return default: - err = badPrefix(UintType, lead) + if isnfixint(lead) { + err = UintBelowZero{Value: int64(rnfixint(lead))} + } else { + err = badPrefix(UintType, lead) + } return } @@ -715,7 +860,7 @@ func (m *Reader) ReadUint() (u uint, err error) { return } -// ReadByte is analagous to ReadUint8. +// ReadByte is analogous to ReadUint8. // // NOTE: this is *not* an implementation // of io.ByteReader. diff --git a/vendor/github.com/tinylib/msgp/msgp/read_bytes.go b/vendor/github.com/tinylib/msgp/msgp/read_bytes.go index 16e7e686..f53f84d0 100644 --- a/vendor/github.com/tinylib/msgp/msgp/read_bytes.go +++ b/vendor/github.com/tinylib/msgp/msgp/read_bytes.go @@ -79,6 +79,9 @@ func (r *Raw) UnmarshalMsg(b []byte) ([]byte, error) { return b, err } rlen := l - len(out) + if IsNil(b[:rlen]) { + rlen = 0 + } if cap(*r) < rlen { *r = make(Raw, rlen) } else { @@ -104,7 +107,11 @@ func (r Raw) EncodeMsg(w *Writer) error { // next object on the wire. func (r *Raw) DecodeMsg(f *Reader) error { *r = (*r)[:0] - return appendNext(f, (*[]byte)(r)) + err := appendNext(f, (*[]byte)(r)) + if IsNil(*r) { + *r = (*r)[:0] + } + return err } // Msgsize implements msgp.Sizer @@ -368,6 +375,15 @@ func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { o = b[2:] return + case muint8: + if l < 2 { + err = ErrShortBytes + return + } + i = int64(getMuint8(b)) + o = b[2:] + return + case mint16: if l < 3 { err = ErrShortBytes @@ -377,6 +393,15 @@ func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { o = b[3:] return + case muint16: + if l < 3 { + err = ErrShortBytes + return + } + i = int64(getMuint16(b)) + o = b[3:] + return + case mint32: if l < 5 { err = ErrShortBytes @@ -386,12 +411,35 @@ func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { o = b[5:] return + case muint32: + if l < 5 { + err = ErrShortBytes + return + } + i = int64(getMuint32(b)) + o = b[5:] + return + case mint64: if l < 9 { err = ErrShortBytes return } - i = getMint64(b) + i = int64(getMint64(b)) + o = b[9:] + return + + case muint64: + if l < 9 { + err = ErrShortBytes + return + } + u := getMuint64(b) + if u > math.MaxInt64 { + err = UintOverflow{Value: u, FailedBitsize: 64} + return + } + i = int64(u) o = b[9:] return @@ -477,6 +525,20 @@ func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { } switch lead { + case mint8: + if l < 2 { + err = ErrShortBytes + return + } + v := int64(getMint8(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[2:] + return + case muint8: if l < 2 { err = ErrShortBytes @@ -486,6 +548,20 @@ func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { o = b[2:] return + case mint16: + if l < 3 { + err = ErrShortBytes + return + } + v := int64(getMint16(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[3:] + return + case muint16: if l < 3 { err = ErrShortBytes @@ -495,6 +571,20 @@ func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { o = b[3:] return + case mint32: + if l < 5 { + err = ErrShortBytes + return + } + v := int64(getMint32(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[5:] + return + case muint32: if l < 5 { err = ErrShortBytes @@ -504,6 +594,20 @@ func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { o = b[5:] return + case mint64: + if l < 9 { + err = ErrShortBytes + return + } + v := int64(getMint64(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[9:] + return + case muint64: if l < 9 { err = ErrShortBytes @@ -514,7 +618,11 @@ func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { return default: - err = badPrefix(UintType, lead) + if isnfixint(lead) { + err = UintBelowZero{Value: int64(rnfixint(lead))} + } else { + err = badPrefix(UintType, lead) + } return } } @@ -576,7 +684,7 @@ func ReadUintBytes(b []byte) (uint, []byte, error) { return uint(u), b, err } -// ReadByteBytes is analagous to ReadUint8Bytes +// ReadByteBytes is analogous to ReadUint8Bytes func ReadByteBytes(b []byte) (byte, []byte, error) { return ReadUint8Bytes(b) } diff --git a/vendor/github.com/tinylib/msgp/msgp/unsafe.go b/vendor/github.com/tinylib/msgp/msgp/unsafe.go index 0cb972e3..3978b6ff 100644 --- a/vendor/github.com/tinylib/msgp/msgp/unsafe.go +++ b/vendor/github.com/tinylib/msgp/msgp/unsafe.go @@ -1,4 +1,4 @@ -// +build !appengine +// +build !purego,!appengine package msgp @@ -24,7 +24,8 @@ const ( // THIS IS EVIL CODE. // YOU HAVE BEEN WARNED. func UnsafeString(b []byte) string { - return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: uintptr(unsafe.Pointer(&b[0])), Len: len(b)})) + sh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: sh.Data, Len: sh.Len})) } // UnsafeBytes returns the string as a byte slice diff --git a/vendor/github.com/tinylib/msgp/msgp/write.go b/vendor/github.com/tinylib/msgp/msgp/write.go index d2700521..fb1947c5 100644 --- a/vendor/github.com/tinylib/msgp/msgp/write.go +++ b/vendor/github.com/tinylib/msgp/msgp/write.go @@ -397,7 +397,7 @@ func (mw *Writer) WriteUint64(u uint64) error { } } -// WriteByte is analagous to WriteUint8 +// WriteByte is analogous to WriteUint8 func (mw *Writer) WriteByte(u byte) error { return mw.WriteUint8(uint8(u)) } // WriteUint8 writes a uint8 to the writer @@ -586,7 +586,7 @@ func (mw *Writer) WriteMapStrIntf(mp map[string]interface{}) (err error) { // elapsed since "zero" Unix time, followed by 4 bytes // for a big-endian 32-bit signed integer denoting // the nanosecond offset of the time. This encoding -// is intended to ease portability accross languages. +// is intended to ease portability across languages. // (Note that this is *not* the standard time.Time // binary encoding, because its implementation relies // heavily on the internal representation used by the @@ -685,11 +685,11 @@ func (mw *Writer) WriteIntf(v interface{}) error { case reflect.Map: return mw.writeMap(val) } - return &ErrUnsupportedType{val.Type()} + return &ErrUnsupportedType{T: val.Type()} } func (mw *Writer) writeMap(v reflect.Value) (err error) { - if v.Elem().Kind() != reflect.String { + if v.Type().Key().Kind() != reflect.String { return errors.New("msgp: map keys must be strings") } ks := v.MapKeys() diff --git a/vendor/github.com/tinylib/msgp/msgp/write_bytes.go b/vendor/github.com/tinylib/msgp/msgp/write_bytes.go index 4f7aa11f..eaa03c46 100644 --- a/vendor/github.com/tinylib/msgp/msgp/write_bytes.go +++ b/vendor/github.com/tinylib/msgp/msgp/write_bytes.go @@ -162,7 +162,7 @@ func AppendUint(b []byte, u uint) []byte { return AppendUint64(b, uint64(u)) } // AppendUint8 appends a uint8 to the slice func AppendUint8(b []byte, u uint8) []byte { return AppendUint64(b, uint64(u)) } -// AppendByte is analagous to AppendUint8 +// AppendByte is analogous to AppendUint8 func AppendByte(b []byte, u byte) []byte { return AppendUint8(b, uint8(u)) } // AppendUint16 appends a uint16 to the slice diff --git a/vendor/gopkg.in/raintank/schema.v1/event.go b/vendor/gopkg.in/raintank/schema.v1/event.go deleted file mode 100644 index e88faa83..00000000 --- a/vendor/gopkg.in/raintank/schema.v1/event.go +++ /dev/null @@ -1,36 +0,0 @@ -package schema - -import ( - "errors" - "fmt" - "strings" -) - -var errInvalidEvent = errors.New("invalid event definition") -var errFmtInvalidSeverity = "invalid severity level %q" - -//go:generate msgp - -type ProbeEvent struct { - Id string `json:"id"` - EventType string `json:"event_type"` - OrgId int64 `json:"org_id"` - Severity string `json:"severity"` // enum "INFO" "WARN" "ERROR" "OK" - Source string `json:"source"` - Timestamp int64 `json:"timestamp"` - Message string `json:"message"` - Tags map[string]string `json:"tags"` -} - -func (e *ProbeEvent) Validate() error { - if e.EventType == "" || e.OrgId == 0 || e.Source == "" || e.Timestamp == 0 || e.Message == "" { - return errInvalidEvent - } - switch strings.ToLower(e.Severity) { - case "info", "ok", "warn", "error", "warning", "critical": - // nop - default: - return fmt.Errorf(errFmtInvalidSeverity, e.Severity) - } - return nil -} diff --git a/vendor/gopkg.in/raintank/schema.v1/event_gen.go b/vendor/gopkg.in/raintank/schema.v1/event_gen.go deleted file mode 100644 index c9c23bc8..00000000 --- a/vendor/gopkg.in/raintank/schema.v1/event_gen.go +++ /dev/null @@ -1,319 +0,0 @@ -package schema - -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT - -import "github.com/tinylib/msgp/msgp" - -// DecodeMsg implements msgp.Decodable -func (z *ProbeEvent) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zbai uint32 - zbai, err = dc.ReadMapHeader() - if err != nil { - return - } - for zbai > 0 { - zbai-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "Id": - z.Id, err = dc.ReadString() - if err != nil { - return - } - case "EventType": - z.EventType, err = dc.ReadString() - if err != nil { - return - } - case "OrgId": - z.OrgId, err = dc.ReadInt64() - if err != nil { - return - } - case "Severity": - z.Severity, err = dc.ReadString() - if err != nil { - return - } - case "Source": - z.Source, err = dc.ReadString() - if err != nil { - return - } - case "Timestamp": - z.Timestamp, err = dc.ReadInt64() - if err != nil { - return - } - case "Message": - z.Message, err = dc.ReadString() - if err != nil { - return - } - case "Tags": - var zcmr uint32 - zcmr, err = dc.ReadMapHeader() - if err != nil { - return - } - if z.Tags == nil && zcmr > 0 { - z.Tags = make(map[string]string, zcmr) - } else if len(z.Tags) > 0 { - for key, _ := range z.Tags { - delete(z.Tags, key) - } - } - for zcmr > 0 { - zcmr-- - var zxvk string - var zbzg string - zxvk, err = dc.ReadString() - if err != nil { - return - } - zbzg, err = dc.ReadString() - if err != nil { - return - } - z.Tags[zxvk] = zbzg - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z *ProbeEvent) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 8 - // write "Id" - err = en.Append(0x88, 0xa2, 0x49, 0x64) - if err != nil { - return err - } - err = en.WriteString(z.Id) - if err != nil { - return - } - // write "EventType" - err = en.Append(0xa9, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65) - if err != nil { - return err - } - err = en.WriteString(z.EventType) - if err != nil { - return - } - // write "OrgId" - err = en.Append(0xa5, 0x4f, 0x72, 0x67, 0x49, 0x64) - if err != nil { - return err - } - err = en.WriteInt64(z.OrgId) - if err != nil { - return - } - // write "Severity" - err = en.Append(0xa8, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79) - if err != nil { - return err - } - err = en.WriteString(z.Severity) - if err != nil { - return - } - // write "Source" - err = en.Append(0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) - if err != nil { - return err - } - err = en.WriteString(z.Source) - if err != nil { - return - } - // write "Timestamp" - err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) - if err != nil { - return err - } - err = en.WriteInt64(z.Timestamp) - if err != nil { - return - } - // write "Message" - err = en.Append(0xa7, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65) - if err != nil { - return err - } - err = en.WriteString(z.Message) - if err != nil { - return - } - // write "Tags" - err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73) - if err != nil { - return err - } - err = en.WriteMapHeader(uint32(len(z.Tags))) - if err != nil { - return - } - for zxvk, zbzg := range z.Tags { - err = en.WriteString(zxvk) - if err != nil { - return - } - err = en.WriteString(zbzg) - if err != nil { - return - } - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *ProbeEvent) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 8 - // string "Id" - o = append(o, 0x88, 0xa2, 0x49, 0x64) - o = msgp.AppendString(o, z.Id) - // string "EventType" - o = append(o, 0xa9, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65) - o = msgp.AppendString(o, z.EventType) - // string "OrgId" - o = append(o, 0xa5, 0x4f, 0x72, 0x67, 0x49, 0x64) - o = msgp.AppendInt64(o, z.OrgId) - // string "Severity" - o = append(o, 0xa8, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79) - o = msgp.AppendString(o, z.Severity) - // string "Source" - o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) - o = msgp.AppendString(o, z.Source) - // string "Timestamp" - o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) - o = msgp.AppendInt64(o, z.Timestamp) - // string "Message" - o = append(o, 0xa7, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65) - o = msgp.AppendString(o, z.Message) - // string "Tags" - o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73) - o = msgp.AppendMapHeader(o, uint32(len(z.Tags))) - for zxvk, zbzg := range z.Tags { - o = msgp.AppendString(o, zxvk) - o = msgp.AppendString(o, zbzg) - } - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *ProbeEvent) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zajw uint32 - zajw, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zajw > 0 { - zajw-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "Id": - z.Id, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - return - } - case "EventType": - z.EventType, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - return - } - case "OrgId": - z.OrgId, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - case "Severity": - z.Severity, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - return - } - case "Source": - z.Source, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - return - } - case "Timestamp": - z.Timestamp, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - case "Message": - z.Message, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - return - } - case "Tags": - var zwht uint32 - zwht, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - if z.Tags == nil && zwht > 0 { - z.Tags = make(map[string]string, zwht) - } else if len(z.Tags) > 0 { - for key, _ := range z.Tags { - delete(z.Tags, key) - } - } - for zwht > 0 { - var zxvk string - var zbzg string - zwht-- - zxvk, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - return - } - zbzg, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - return - } - z.Tags[zxvk] = zbzg - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *ProbeEvent) Msgsize() (s int) { - s = 1 + 3 + msgp.StringPrefixSize + len(z.Id) + 10 + msgp.StringPrefixSize + len(z.EventType) + 6 + msgp.Int64Size + 9 + msgp.StringPrefixSize + len(z.Severity) + 7 + msgp.StringPrefixSize + len(z.Source) + 10 + msgp.Int64Size + 8 + msgp.StringPrefixSize + len(z.Message) + 5 + msgp.MapHeaderSize - if z.Tags != nil { - for zxvk, zbzg := range z.Tags { - _ = zbzg - s += msgp.StringPrefixSize + len(zxvk) + msgp.StringPrefixSize + len(zbzg) - } - } - return -} diff --git a/vendor/gopkg.in/raintank/schema.v1/metric.go b/vendor/gopkg.in/raintank/schema.v1/metric.go deleted file mode 100644 index 85473244..00000000 --- a/vendor/gopkg.in/raintank/schema.v1/metric.go +++ /dev/null @@ -1,204 +0,0 @@ -package schema - -import ( - "bytes" - "crypto/md5" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "sort" -) - -var errInvalidIntervalzero = errors.New("interval cannot be 0") -var errInvalidOrgIdzero = errors.New("org-id cannot be 0") -var errInvalidEmptyName = errors.New("name cannot be empty") -var errInvalidEmptyMetric = errors.New("metric cannot be empty") -var errInvalidMtype = errors.New("invalid mtype") - -type PartitionedMetric interface { - Validate() error - SetId() - // return a []byte key comprised of the metric's OrgId - // accepts an input []byte to allow callers to re-use - // buffers to reduce memory allocations - KeyByOrgId([]byte) []byte - // return a []byte key comprised of the metric's Name - // accepts an input []byte to allow callers to re-use - // buffers to reduce memory allocations - KeyBySeries([]byte) []byte -} - -//go:generate msgp - -// MetricData contains all metric metadata (some as fields, some as tags) and a datapoint -type MetricData struct { - Id string `json:"id"` - OrgId int `json:"org_id"` - Name string `json:"name"` - Metric string `json:"metric"` - Interval int `json:"interval"` - Value float64 `json:"value"` - Unit string `json:"unit"` - Time int64 `json:"time"` - Mtype string `json:"mtype"` - Tags []string `json:"tags" elastic:"type:string,index:not_analyzed"` -} - -func (m *MetricData) Validate() error { - if m.OrgId == 0 { - return errInvalidOrgIdzero - } - if m.Interval == 0 { - return errInvalidIntervalzero - } - if m.Name == "" { - return errInvalidEmptyName - } - if m.Metric == "" { - return errInvalidEmptyMetric - } - if m.Mtype == "" || (m.Mtype != "gauge" && m.Mtype != "rate" && m.Mtype != "count" && m.Mtype != "counter" && m.Mtype != "timestamp") { - return errInvalidMtype - } - return nil -} - -func (m *MetricData) KeyByOrgId(b []byte) []byte { - if cap(b)-len(b) < 4 { - // not enough unused space in the slice so we need to grow it. - newBuf := make([]byte, len(b), len(b)+4) - copy(newBuf, b) - b = newBuf - } - // PutUint32 writes directly to the slice rather then appending. - // so we need to set the length to 4 more bytes then it currently is. - b = b[:len(b)+4] - binary.LittleEndian.PutUint32(b[len(b)-4:], uint32(m.OrgId)) - return b -} - -func (m *MetricData) KeyBySeries(b []byte) []byte { - b = append(b, []byte(m.Name)...) - return b -} - -// returns a id (hash key) in the format OrgId.md5Sum -// the md5sum is a hash of the the concatination of the -// metric + each tag key:value pair (in metrics2.0 sense, so also fields), sorted alphabetically. -func (m *MetricData) SetId() { - sort.Strings(m.Tags) - - buffer := bytes.NewBufferString(m.Metric) - buffer.WriteByte(0) - buffer.WriteString(m.Unit) - buffer.WriteByte(0) - buffer.WriteString(m.Mtype) - buffer.WriteByte(0) - fmt.Fprintf(buffer, "%d", m.Interval) - - for _, k := range m.Tags { - buffer.WriteByte(0) - buffer.WriteString(k) - } - m.Id = fmt.Sprintf("%d.%x", m.OrgId, md5.Sum(buffer.Bytes())) -} - -// can be used by some encoders, such as msgp -type MetricDataArray []*MetricData - -// for ES -type MetricDefinition struct { - Id string `json:"id"` - OrgId int `json:"org_id"` - Name string `json:"name" elastic:"type:string,index:not_analyzed"` // graphite format - Metric string `json:"metric"` // kairosdb format (like graphite, but not including some tags) - Interval int `json:"interval"` // minimum 10 - Unit string `json:"unit"` - Mtype string `json:"mtype"` - Tags []string `json:"tags" elastic:"type:string,index:not_analyzed"` - LastUpdate int64 `json:"lastUpdate"` // unix timestamp - Partition int32 `json:"partition"` -} - -func (m *MetricDefinition) SetId() { - sort.Strings(m.Tags) - - buffer := bytes.NewBufferString(m.Metric) - buffer.WriteByte(0) - buffer.WriteString(m.Unit) - buffer.WriteByte(0) - buffer.WriteString(m.Mtype) - buffer.WriteByte(0) - fmt.Fprintf(buffer, "%d", m.Interval) - - for _, k := range m.Tags { - buffer.WriteByte(0) - buffer.WriteString(k) - } - m.Id = fmt.Sprintf("%d.%x", m.OrgId, md5.Sum(buffer.Bytes())) -} - -func (m *MetricDefinition) Validate() error { - if m.OrgId == 0 { - return errInvalidOrgIdzero - } - if m.Interval == 0 { - return errInvalidIntervalzero - } - if m.Name == "" { - return errInvalidEmptyName - } - if m.Metric == "" { - return errInvalidEmptyMetric - } - if m.Mtype == "" || (m.Mtype != "gauge" && m.Mtype != "rate" && m.Mtype != "count" && m.Mtype != "counter" && m.Mtype != "timestamp") { - return errInvalidMtype - } - return nil -} - -func (m *MetricDefinition) KeyByOrgId(b []byte) []byte { - if cap(b)-len(b) < 4 { - // not enough unused space in the slice so we need to grow it. - newBuf := make([]byte, len(b), len(b)+4) - copy(newBuf, b) - b = newBuf - } - // PutUint32 writes directly to the slice rather then appending. - // so we need to set the length to 4 more bytes then it currently is. - b = b[:len(b)+4] - binary.LittleEndian.PutUint32(b[len(b)-4:], uint32(m.OrgId)) - return b -} - -func (m *MetricDefinition) KeyBySeries(b []byte) []byte { - b = append(b, []byte(m.Name)...) - return b -} - -func MetricDefinitionFromJSON(b []byte) (*MetricDefinition, error) { - def := new(MetricDefinition) - if err := json.Unmarshal(b, &def); err != nil { - return nil, err - } - return def, nil -} - -// MetricDefinitionFromMetricData yields a MetricDefinition that has no references -// to the original MetricData -func MetricDefinitionFromMetricData(d *MetricData) *MetricDefinition { - tags := make([]string, len(d.Tags)) - copy(tags, d.Tags) - return &MetricDefinition{ - Id: d.Id, - Name: d.Name, - OrgId: d.OrgId, - Metric: d.Metric, - Mtype: d.Mtype, - Interval: d.Interval, - LastUpdate: d.Time, - Unit: d.Unit, - Tags: tags, - } -} diff --git a/vendor/gopkg.in/raintank/schema.v1/msg/format_string.go b/vendor/gopkg.in/raintank/schema.v1/msg/format_string.go deleted file mode 100644 index 7749e61f..00000000 --- a/vendor/gopkg.in/raintank/schema.v1/msg/format_string.go +++ /dev/null @@ -1,16 +0,0 @@ -// generated by stringer -type=Format; DO NOT EDIT - -package msg - -import "fmt" - -const _Format_name = "FormatMetricDataArrayJsonFormatMetricDataArrayMsgp" - -var _Format_index = [...]uint8{0, 25, 50} - -func (i Format) String() string { - if i >= Format(len(_Format_index)-1) { - return fmt.Sprintf("Format(%d)", i) - } - return _Format_name[_Format_index[i]:_Format_index[i+1]] -} diff --git a/vendor/gopkg.in/raintank/schema.v1/msg/msg.go b/vendor/gopkg.in/raintank/schema.v1/msg/msg.go deleted file mode 100644 index f6af779c..00000000 --- a/vendor/gopkg.in/raintank/schema.v1/msg/msg.go +++ /dev/null @@ -1,204 +0,0 @@ -package msg - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/codeskyblue/go-uuid" - "gopkg.in/raintank/schema.v1" -) - -var errTooSmall = errors.New("too small") -var errFmtBinWriteFailed = "binary write failed: %q" -var errFmtUnknownFormat = "unknown format %d" - -type MetricData struct { - Id int64 - Metrics []*schema.MetricData - Produced time.Time - Format Format - Msg []byte -} - -type ProbeEvent struct { - Id int64 - Produced time.Time - Event *schema.ProbeEvent - Format Format - Msg []byte -} - -type ProbeEventJson struct { - Id string `json:"id"` - EventType string `json:"event_type"` - OrgId int64 `json:"org_id"` - Severity string `json:"severity"` - Source string `json:"source"` - Timestamp int64 `json:"timestamp"` - Message string `json:"message"` - Tags []string `json:"tags"` -} - -// parses format and id (cheap), but doesn't decode metrics (expensive) just yet. -func (m *MetricData) InitFromMsg(msg []byte) error { - if len(msg) < 9 { - return errTooSmall - } - m.Msg = msg - - buf := bytes.NewReader(msg[1:9]) - binary.Read(buf, binary.BigEndian, &m.Id) - m.Produced = time.Unix(0, m.Id) - - m.Format = Format(msg[0]) - if m.Format != FormatMetricDataArrayJson && m.Format != FormatMetricDataArrayMsgp { - return fmt.Errorf(errFmtUnknownFormat, m.Format) - } - return nil -} - -// sets m.Metrics to a []*schema.MetricData -// any subsequent call may however put different MetricData into our m.Metrics array -func (m *MetricData) DecodeMetricData() error { - var err error - switch m.Format { - case FormatMetricDataArrayJson: - err = json.Unmarshal(m.Msg[9:], &m.Metrics) - case FormatMetricDataArrayMsgp: - out := schema.MetricDataArray(m.Metrics) - _, err = out.UnmarshalMsg(m.Msg[9:]) - m.Metrics = []*schema.MetricData(out) - default: - return fmt.Errorf("unrecognized format %d", m.Msg[0]) - } - if err != nil { - return fmt.Errorf("ERROR: failure to unmarshal message body via format %q: %s", m.Format, err) - } - m.Msg = nil // no more need for the original input - return nil -} - -func CreateMsg(metrics []*schema.MetricData, id int64, version Format) ([]byte, error) { - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, uint8(version)) - if err != nil { - return nil, fmt.Errorf(errFmtBinWriteFailed, err) - } - err = binary.Write(buf, binary.BigEndian, id) - if err != nil { - return nil, fmt.Errorf(errFmtBinWriteFailed, err) - } - var msg []byte - switch version { - case FormatMetricDataArrayJson: - msg, err = json.Marshal(metrics) - case FormatMetricDataArrayMsgp: - m := schema.MetricDataArray(metrics) - msg, err = m.MarshalMsg(nil) - default: - return nil, fmt.Errorf(errFmtUnknownFormat, version) - } - if err != nil { - return nil, fmt.Errorf("Failed to marshal metrics payload: %s", err) - } - _, err = buf.Write(msg) - if err != nil { - return nil, fmt.Errorf(errFmtBinWriteFailed, err) - } - return buf.Bytes(), nil -} - -func ProbeEventFromMsg(msg []byte) (*ProbeEvent, error) { - e := &ProbeEvent{ - Event: &schema.ProbeEvent{}, - Msg: msg, - } - if len(msg) < 9 { - return e, errTooSmall - } - - buf := bytes.NewReader(msg[1:9]) - binary.Read(buf, binary.BigEndian, &e.Id) - e.Produced = time.Unix(0, e.Id) - - format := Format(msg[0]) - if format != FormatProbeEventJson && format != FormatProbeEventMsgp { - return e, fmt.Errorf(errFmtUnknownFormat, format) - } - e.Format = format - return e, nil -} - -func (e *ProbeEvent) DecodeProbeEvent() error { - var err error - switch e.Format { - case FormatProbeEventJson: - oldFormat := &ProbeEventJson{} - err = json.Unmarshal(e.Msg[9:], oldFormat) - //convert our []string of key:valy pairs to - // map[string]string - tags := make(map[string]string) - for _, t := range oldFormat.Tags { - parts := strings.SplitN(t, ":", 2) - tags[parts[0]] = parts[1] - } - e.Event = &schema.ProbeEvent{ - Id: oldFormat.Id, - EventType: oldFormat.EventType, - OrgId: oldFormat.OrgId, - Severity: oldFormat.Severity, - Source: oldFormat.Source, - Timestamp: oldFormat.Timestamp, - Message: oldFormat.Message, - Tags: tags, - } - case FormatProbeEventMsgp: - _, err = e.Event.UnmarshalMsg(e.Msg[9:]) - default: - return fmt.Errorf(errFmtUnknownFormat, e.Msg[0]) - } - if err != nil { - return fmt.Errorf("ERROR: failure to unmarshal message body via format %q: %s", e.Format, err) - } - return nil -} - -func CreateProbeEventMsg(event *schema.ProbeEvent, id int64, version Format) ([]byte, error) { - if event.Id == "" { - // per http://blog.mikemccandless.com/2014/05/choosing-fast-unique-identifier-uuid.html, - // using V1 UUIDs is much faster than v4 like we were using - u := uuid.NewUUID() - event.Id = u.String() - } - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, uint8(version)) - if err != nil { - return nil, fmt.Errorf(errFmtBinWriteFailed, err) - } - err = binary.Write(buf, binary.BigEndian, id) - if err != nil { - return nil, fmt.Errorf(errFmtBinWriteFailed, err) - } - var msg []byte - switch version { - case FormatProbeEventJson: - msg, err = json.Marshal(event) - case FormatProbeEventMsgp: - msg, err = event.MarshalMsg(nil) - default: - return nil, fmt.Errorf(errFmtUnknownFormat, version) - } - if err != nil { - return nil, fmt.Errorf("Failed to marshal metrics payload: %s", err) - } - _, err = buf.Write(msg) - if err != nil { - return nil, fmt.Errorf(errFmtBinWriteFailed, err) - } - return buf.Bytes(), nil -}