Refactor releases

.circleci/config.yml

@@ -3,7 +3,7 @@ jobs:
   build:
     working_directory: /go/src/github.com/grafana/metrictank
     docker:
-      - image: circleci/golang:1.11
+      - image: circleci/golang:1.11.4
     steps:
       - checkout
       - run: scripts/build.sh
@@ -21,15 +21,15 @@ jobs:
   test:
     working_directory: /go/src/github.com/grafana/metrictank
     docker:
-      - image: circleci/golang:1.11
+      - image: circleci/golang:1.11.4
     steps:
       - checkout
       - run: go test -v -race $(go list ./... | grep -v github.com/grafana/metrictank/stacktest)
 
   qa:
     working_directory: /go/src/github.com/grafana/metrictank
     docker:
-      - image: circleci/golang:1.11
+      - image: circleci/golang:1.11.4
     steps:
       - checkout
       - run: scripts/qa/gofmt.sh

README.md

@@ -55,7 +55,7 @@ Otherwise data loss of current chunks will be incurred.  See [operations guide](
 
 * [Overview](https://github.com/grafana/metrictank/blob/master/docs/overview.md)
 * [Quick start using docker](https://github.com/grafana/metrictank/blob/master/docs/quick-start-docker.md)
-  ([other docker stacks](docs/docker.md))
+  ([more docker information](docs/docker.md))
 * [Installation guides](https://github.com/grafana/metrictank/blob/master/docs/installation.md)
 * [Configuration](https://github.com/grafana/metrictank/blob/master/docs/config.md)
 * [Memory server](https://github.com/grafana/metrictank/blob/master/docs/memory-server.md)

cmd/metrictank/metrictank.go

@@ -44,7 +44,7 @@ import (
 var (
 	warmupPeriod time.Duration
 	startupTime  time.Time
-	gitHash      = "(none)"
+	version      = "(none)"
 
 	metrics     *mdata.AggMetrics
 	metricIndex idx.MetricIndex
@@ -88,7 +88,7 @@ func main() {
 
 	// if the user just wants the version, give it and exit
 	if *showVersion {
-		fmt.Printf("metrictank (built with %s, git hash %s)\n", runtime.Version(), gitHash)
+		fmt.Printf("metrictank (version: %s - runtime: %s)\n", version, runtime.Version())
 		return
 	}
 
@@ -170,7 +170,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Could not parse port from listenAddr. %s", api.Addr)
 	}
-	cluster.Init(*instance, gitHash, startupTime, scheme, int(port))
+	cluster.Init(*instance, version, startupTime, scheme, int(port))
 
 	/***********************************
 		Validate remaining settings
@@ -226,9 +226,9 @@ func main() {
 	/***********************************
 		Report Version
 	***********************************/
-	log.Infof("Metrictank starting. Built from %s - Go version %s", gitHash, runtime.Version())
+	log.Infof("Metrictank starting. version: %s - runtime: %s", version, runtime.Version())
 	// metric version.%s is the version of metrictank running.  The metric value is always 1
-	mtVersion := stats.NewBool(fmt.Sprintf("version.%s", strings.Replace(gitHash, ".", "_", -1)))
+	mtVersion := stats.NewBool(fmt.Sprintf("version.%s", strings.Replace(version, ".", "_", -1)))
 	mtVersion.Set(true)
 
 	/***********************************

cmd/mt-aggs-explain/main.go

@@ -13,7 +13,7 @@ import (
 )
 
 var (
-	gitHash     = "(none)"
+	version     = "(none)"
 	showVersion = flag.Bool("version", false, "print version string")
 	metric      = flag.String("metric", "", "specify a metric name to see which aggregation rule it matches")
 )
@@ -40,7 +40,7 @@ func main() {
 	flag.Parse()
 
 	if *showVersion {
-		fmt.Printf("mt-aggs-explain (built with %s, git hash %s)\n", runtime.Version(), gitHash)
+		fmt.Printf("mt-aggs-explain (version: %s - runtime: %s)\n", version, runtime.Version())
 		return
 	}
 	if flag.NArg() > 1 {

cmd/mt-schemas-explain/main.go

@@ -14,7 +14,7 @@ import (
 )
 
 var (
-	gitHash      = "(none)"
+	version      = "(none)"
 	showVersion  = flag.Bool("version", false, "print version string")
 	windowFactor = flag.Int("window-factor", 20, "size of compaction window relative to TTL")
 	metric       = flag.String("metric", "", "specify a metric name to see which schema it matches")
@@ -43,7 +43,7 @@ func main() {
 	flag.Parse()
 
 	if *showVersion {
-		fmt.Printf("mt-schemas-explain (built with %s, git hash %s)\n", runtime.Version(), gitHash)
+		fmt.Printf("mt-schemas-explain (version: %s - runtime: %s)\n", version, runtime.Version())
 		return
 	}
 	if flag.NArg() > 1 {

cmd/mt-store-cat/main.go

@@ -22,7 +22,7 @@ import (
 const tsFormat = "2006-01-02 15:04:05"
 
 var (
-	gitHash = "(none)"
+	version = "(none)"
 
 	// flags from metrictank.go, globals
 	showVersion = flag.Bool("version", false, "print version string")
@@ -111,7 +111,7 @@ func main() {
 	flag.Parse()
 
 	if *showVersion {
-		fmt.Printf("mt-store-cat (built with %s, git hash %s)\n", runtime.Version(), gitHash)
+		fmt.Printf("mt-store-cat (version: %s - runtime: %s)\n", version, runtime.Version())
 		return
 	}
 	if flag.NArg() < 1 {
@@ -157,11 +157,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	if *showVersion {
-		fmt.Printf("mt-store-cat (built with %s, git hash %s)\n", runtime.Version(), gitHash)
-		return
-	}
-
 	printTime = printTimeFormatted
 	if *printTs {
 		printTime = printTimeUnix

cmd/mt-view-boundaries/main.go

@@ -14,7 +14,7 @@ import (
 )
 
 var (
-	gitHash     string
+	version     string
 	showVersion = flag.Bool("version", false, "print version string")
 	spanStr     = flag.String("span", "", "see boundaries for chunks of this span")
 	now         = time.Now()
@@ -58,7 +58,7 @@ func main() {
 	flag.Parse()
 
 	if *showVersion {
-		fmt.Printf("mt-view-boundaries (built with %s, git hash %s)\n", runtime.Version(), gitHash)
+		fmt.Printf("mt-view-boundaries (version: %s - runtime: %s)\n", version, runtime.Version())
 		return
 	}
 

cmd/mt-whisper-importer-writer/main.go

@@ -68,7 +68,7 @@ var (
 		"If true existing chunks may be overwritten",
 	)
 
-	gitHash = "(none)"
+	version = "(none)"
 )
 
 type Server struct {
@@ -180,7 +180,7 @@ func main() {
 		panic(fmt.Sprintf("Failed to instantiate partitioner: %q", err))
 	}
 
-	cluster.Init("mt-whisper-importer-writer", gitHash, time.Now(), "http", int(80))
+	cluster.Init("mt-whisper-importer-writer", version, time.Now(), "http", int(80))
 
 	server := &Server{
 		Session:     store.Session,

docs/CONTRIBUTING.md

@@ -48,3 +48,13 @@ see the [Makefile](../Makefile) for more targets
 * [docs](../docs) for users, operators and basic development instructions
 * [devdocs](../devdocs) in-depth implementation details for developers
 
+# Release process
+
+During normal development, maintain CHANGELOG.md, and mark interesting -to users- changes under "unreleased" version.
+Grafana Labs regularly deploys the latest code from `master`, but cannot possibly do extensive testing of all functionality in production, so users are encouraged to run master also, and report any issues they hit.
+When significant changes have been merged to master, and they have had a chance to be tested or run in production for a while, we tag a release, as follows:
+
+* update CHANGELOG.md from `unreleased` to the version.
+* create annotated git tag in the form `v<version>` and push to GitHub
+* wait for CircleCI to complete successfully.
+* create release on GitHub. copy entry from CHANGELOG.md to GitHub release page

docs/docker.md

@@ -1,4 +1,15 @@
-# docker stacks
+# docker
+
+## tags
+
+[metrictank on dockerhub](https://hub.docker.com/r/grafana/metrictank/)
+
+You can use these tags:
+
+* `latest`: the latest official stable release, corresponding to a git tag.
+* `master`: latest build from master. these versions typically bring improvements but possibly also new bugs
+
+## extra docker stacks
 
 You may have already followed [our docker quickstart guide](https://github.com/grafana/metrictank/blob/master/docs/quick-start-docker.md) but in fact,
 the metrictank repository contains a variety of docker stacks for different purposes.

docs/faq.md

@@ -21,7 +21,6 @@ Metrictank will automatically combine the data together like so:
 
 ## Tag support
 
-Metrictank aims to provide a proper implementation of [metrics 2.0](http://metrics20.org/)
+Metrics can have tags and metrictank supports queries via the [graphite tag query api](https://graphite.readthedocs.io/en/latest/tags.html)
+In the future, metrictank aims to provide a proper implementation of [metrics 2.0](http://metrics20.org/)
 (it helps that both projects share the same main author)
-As such, metrics can have tags and they are indexed as such.  However we don't expose this yet through the search or query interface,
-but plan to do so in the future.

docs/installation-deb.md

@@ -34,34 +34,32 @@ You need root access. All the commands shown assume you're root.
 
 ## Metrictank and graphite
 
-We automatically build rpms and debs on circleCi for all needed components whenever the build succeeds.
-These packages are pushed to packagecloud.
+Grafana Labs provides 2 repositories:
 
-You need to install these packages:
-
-* metrictank
+* [raintank](https://packagecloud.io/raintank/raintank): stable repository for [official stable releases](https://github.com/grafana/metrictank/releases)
+* [testing](https://packagecloud.io/raintank/testing): testing repository that has the latest packages which typically bring improvements but possibly also new bugs. These packages are built from git master and are named `<base-release>-<commits-since-release>-<git-hash>`
 
-Releases are simply tagged versions like `0.5.1` ([releases](https://github.com/grafana/metrictank/releases)),
-whereas commits in master following a release will be named `version-commit-after` for example `0.5.1-20` for
-the 20th commit after `0.5.1`
-
-We aim to keep master stable, so that's your best bet.
+See the installation instructions on those pages for how to enable the repositories for your distribution
 
 Supported distributions:
 
 * Ubuntu 14.04 (Trusty Tahr), 16.04 (Xenial Xerus)
 * Debian 7 (wheezy), 8 (jessie)
 * Centos 6, 7
 
+You need to install these packages:
+
+* metrictank
+
+
 ### Install Metrictank
 You can enable our repository and install the metrictank package like so:
+(Feel free to use the testing repository instead)
 
 ```
 curl -s https://packagecloud.io/install/repositories/raintank/raintank/script.deb.sh | bash
 apt-get install metrictank
 ```
-[more info](https://packagecloud.io/raintank/raintank/install)
-
 
 ### Install Graphite
 

docs/installation-rpm.md

@@ -34,34 +34,32 @@ You need root access. All the commands shown assume you're root.
 
 ## Metrictank and graphite
 
-We automatically build rpms and debs on circleCi for all needed components whenever the build succeeds.
-These packages are pushed to packagecloud.
+Grafana Labs provides 2 repositories:
 
-You need to install these packages:
-
-* metrictank
+* [raintank](https://packagecloud.io/raintank/raintank): stable repository for [official stable releases](https://github.com/grafana/metrictank/releases)
+* [testing](https://packagecloud.io/raintank/testing): testing repository that has the latest packages which typically bring improvements but possibly also new bugs. These packages are built from git master and are named `<base-release>-<commits-since-release>-<git-hash>`
 
-Releases are simply tagged versions like `0.5.1` ([releases](https://github.com/grafana/metrictank/releases)),
-whereas commits in master following a release will be named `version-commit-after` for example `0.5.1-20` for
-the 20th commit after `0.5.1`
-
-We aim to keep master stable, so that's your best bet.
+See the installation instructions on those pages for how to enable the repositories for your distribution
 
 Supported distributions:
 
 * Ubuntu 14.04 (Trusty Tahr), 16.04 (Xenial Xerus)
 * Debian 7 (wheezy), 8 (jessie)
 * Centos 6, 7
 
+You need to install these packages:
+
+* metrictank
+
+
 ### Install Metrictank
 You can enable our repository and install the metrictank package like so:
+(Feel free to use the testing repository instead)
 
 ```
 curl -s https://packagecloud.io/install/repositories/raintank/raintank/script.rpm.sh | bash
 apt-get install metrictank
 ```
-[more info](https://packagecloud.io/raintank/raintank/install)
-
 
 ### Install Graphite
 

docs/quick-start-docker-cluster.md

@@ -16,7 +16,7 @@ The docker-cluster image is similar to docker-standard, but with the following c
 * Short chunkspan & numchunks (benefits: easy to trigger mem and mem_and_cass requests, frequent cass saves, notifier messages etc)
 * Prometheus monitoring
 * Jaeger tracing
-* caddy-proxied endpooints/datasources to check with different org-id auth headers
+* caddy-proxied endpoints/datasources to check with different org-id auth headers
 * tsdb-gw
 
 

docs/tags.md

@@ -1,9 +1,13 @@
 # Tags
 
-Metrictank uses [metrics2.0](http://metrics20.org/) for structuring metrics and naming them.
+Metrictank implements tag ingestion, storage, and querying to be compatible with the [graphite tags feature](https://graphite.readthedocs.io/en/latest/tags.html).
+
+# Future Metrics2.0 plans
+
+[metrics2.0](http://metrics20.org/)
 Note that tags are modeled as an array of strings, where you can have `key:value` tags but also simply `value` tags.
 
-While metrictank can ingest and store data in metrics2.0 format, making use out of this data is still on ongoing project.
+We want to go further with our tag support and adopt metrics2.0 fully, which brings naming conventions and leveraging semantics.
 
 Here are some goals:
 

scripts/build.sh

@@ -11,7 +11,8 @@ cd ${DIR}/..
 # and https://stackoverflow.com/questions/37531605/how-to-test-if-git-repository-is-shallow
 [ -f $(git rev-parse --git-dir)/shallow ] && git fetch --unshallow
 
-GITVERSION=`git describe --abbrev=7`
+source scripts/version-tag.sh
+
 BUILDDIR=$(pwd)/build
 
 # Make dir
@@ -36,10 +37,10 @@ for bin in *; do
   then
     set -x
     # -race requires CGO
-    CGO_ENABLED=1 go build -race -ldflags "-X main.gitHash=$GITVERSION" -o $BUILDDIR/$bin || fail
+    CGO_ENABLED=1 go build -race -ldflags "-X main.version=$version" -o $BUILDDIR/$bin || fail
   else
     set -x
-    go build -ldflags "-X main.gitHash=$GITVERSION" -o $BUILDDIR/$bin || fail
+    go build -ldflags "-X main.version=$version" -o $BUILDDIR/$bin || fail
   fi
   set +x
   cd ..

scripts/build_docker.sh

@@ -4,20 +4,19 @@ set -x
 # Find the directory we exist within
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 cd ${DIR}
-
-VERSION=`git describe --abbrev=7`
+source version-tag.sh
 
 # regular image
 rm -rf build/*
 mkdir -p build
 cp ../build/* build/
 
 docker build -t grafana/metrictank .
-docker tag grafana/metrictank grafana/metrictank:latest
-docker tag grafana/metrictank grafana/metrictank:$VERSION
+docker tag grafana/metrictank grafana/metrictank:$tag
+docker tag grafana/metrictank grafana/metrictank:$version
 
 # k8s image
 cd ${DIR}/k8s
 docker build -t us.gcr.io/metrictank-gcr/metrictank .
-docker tag us.gcr.io/metrictank-gcr/metrictank us.gcr.io/metrictank-gcr/metrictank:latest
-docker tag us.gcr.io/metrictank-gcr/metrictank us.gcr.io/metrictank-gcr/metrictank:${VERSION}
+docker tag us.gcr.io/metrictank-gcr/metrictank us.gcr.io/metrictank-gcr/metrictank:$tag
+docker tag us.gcr.io/metrictank-gcr/metrictank us.gcr.io/metrictank-gcr/metrictank:$version