api: Refactor API into a new package

CHANGELOG.md

@@ -26,15 +26,13 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 - [#2832](https://github.com/thanos-io/thanos/pull/2832) ui: React: Add runtime and build info page
 - [#2305](https://github.com/thanos-io/thanos/pull/2305) Receive,Sidecar,Ruler: Propagate correct (stricter) MinTime for no-block TSDBs.
+- [#2926](https://github.com/thanos-io/thanos/pull/2926) API: Add new blocks HTTP API to serve blocks metadata. The status endpoints (`/api/v1/status/flags`, `/api/v1/status/runtimeinfo` and `/api/v1/status/buildinfo`) are now available on all components with a HTTP API.
 - [#2892](https://github.com/thanos-io/thanos/pull/2892) Receive: Receiver fails when the initial upload fails.
 
 ### Changed
 
 - [#2893](https://github.com/thanos-io/thanos/pull/2893) Store: Rename metric `thanos_bucket_store_cached_postings_compression_time_seconds` to `thanos_bucket_store_cached_postings_compression_time_seconds_total`.
 - [#2915](https://github.com/thanos-io/thanos/pull/2915) Receive,Ruler: Enable TSDB directory locking by default. Add a new flag (`--tsdb.no-lockfile`) to override behavior.
-
-### Changed
-
 - [#2902](https://github.com/thanos-io/thanos/pull/2902) ui: React: Separate dedupe and partial response checkboxes per panel.
 
 ## [v0.14.0](https://github.com/thanos-io/thanos/releases/tag/v0.14.0) - 2020.07.10

cmd/thanos/compact.go

@@ -22,7 +22,9 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/route"
 	"github.com/prometheus/prometheus/tsdb"
+	blocksAPI "github.com/thanos-io/thanos/pkg/api/blocks"
 	"github.com/thanos-io/thanos/pkg/block"
+	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/compact"
 	"github.com/thanos-io/thanos/pkg/compact/downsample"
 	"github.com/thanos-io/thanos/pkg/component"
@@ -81,16 +83,20 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) {
 	conf.registerFlag(cmd)
 
 	m[component.Compact.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
-		return runCompact(g, logger, reg, component.Compact, *conf)
+		flagsMap := getFlagsMap(cmd.Model().Flags)
+
+		return runCompact(g, logger, tracer, reg, component.Compact, *conf, flagsMap)
 	}
 }
 
 func runCompact(
 	g *run.Group,
 	logger log.Logger,
+	tracer opentracing.Tracer,
 	reg *prometheus.Registry,
 	component component.Component,
 	conf compactConfig,
+	flagsMap map[string]string,
 ) error {
 	deleteDelay := time.Duration(conf.deleteDelay)
 	halted := promauto.With(reg).NewGauge(prometheus.GaugeOpts{
@@ -393,10 +399,16 @@ func runCompact(
 		global := ui.NewBucketUI(logger, conf.label, path.Join(conf.webConf.externalPrefix, "/global"), conf.webConf.prefixHeaderName)
 		global.Register(r, ins)
 
+		api := blocksAPI.NewBlocksAPI(logger, conf.label, flagsMap)
+		api.Register(r.WithPrefix("/api/v1"), tracer, logger, ins)
+
 		// Separate fetcher for global view.
 		// TODO(bwplotka): Allow Bucket UI to visualize the state of the block as well.
 		f := baseMetaFetcher.NewMetaFetcher(extprom.WrapRegistererWithPrefix("thanos_bucket_ui", reg), nil, nil, "component", "globalBucketUI")
-		f.UpdateOnChange(global.Set)
+		f.UpdateOnChange(func(blocks []metadata.Meta, err error) {
+			global.Set(blocks, err)
+			api.Set(blocks, err)
+		})
 
 		srv.Handle("/", r)
 

cmd/thanos/main.go

@@ -243,3 +243,19 @@ func reload(logger log.Logger, cancel <-chan struct{}, r chan<- struct{}) error
 		}
 	}
 }
+
+func getFlagsMap(flags []*kingpin.FlagModel) map[string]string {
+	flagsMap := map[string]string{}
+
+	// Exclude kingpin default flags to expose only Thanos ones.
+	boilerplateFlags := kingpin.New("", "").Version("")
+
+	for _, f := range flags {
+		if boilerplateFlags.GetFlag(f.Name) != nil {
+			continue
+		}
+		flagsMap[f.Name] = f.Value.String()
+	}
+
+	return flagsMap
+}

cmd/thanos/query.go

@@ -8,8 +8,6 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"os"
-	"runtime"
 	"strings"
 	"time"
 
@@ -21,13 +19,13 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/common/route"
-	"github.com/prometheus/common/version"
 	"github.com/prometheus/prometheus/discovery/file"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/promql"
 	"gopkg.in/alecthomas/kingpin.v2"
 
+	v1 "github.com/thanos-io/thanos/pkg/api/query"
 	"github.com/thanos-io/thanos/pkg/component"
 	"github.com/thanos-io/thanos/pkg/discovery/cache"
 	"github.com/thanos-io/thanos/pkg/discovery/dns"
@@ -36,7 +34,6 @@ import (
 	extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
 	"github.com/thanos-io/thanos/pkg/prober"
 	"github.com/thanos-io/thanos/pkg/query"
-	v1 "github.com/thanos-io/thanos/pkg/query/api"
 	"github.com/thanos-io/thanos/pkg/rules"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	grpcserver "github.com/thanos-io/thanos/pkg/server/grpc"
@@ -152,18 +149,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) {
 
 		promql.SetDefaultEvaluationInterval(time.Duration(*defaultEvaluationInterval))
 
-		flagsMap := map[string]string{}
-
-		// Exclude kingpin default flags to expose only Thanos ones.
-		boilerplateFlags := kingpin.New("", "").Version("")
-
-		for _, f := range cmd.Model().Flags {
-			if boilerplateFlags.GetFlag(f.Name) != nil {
-				continue
-			}
-
-			flagsMap[f.Name] = f.Value.String()
-		}
+		flagsMap := getFlagsMap(cmd.Model().Flags)
 
 		return runQuery(
 			g,
@@ -415,39 +401,11 @@ func runQuery(
 			router = router.WithPrefix(webRoutePrefix)
 		}
 
-		buildInfo := &v1.ThanosVersion{
-			Version:   version.Version,
-			Revision:  version.Revision,
-			Branch:    version.Branch,
-			BuildUser: version.BuildUser,
-			BuildDate: version.BuildDate,
-			GoVersion: version.GoVersion,
-		}
-
-		CWD, err := os.Getwd()
-		if err != nil {
-			CWD = "<error retrieving current working directory>"
-			level.Warn(logger).Log("msg", "failed to retrieve current working directory", "err", err)
-		}
-
-		birth := time.Now()
-
-		var runtimeInfo v1.RuntimeInfoFn = func() v1.RuntimeInfo {
-			return v1.RuntimeInfo{
-				StartTime:      birth,
-				CWD:            CWD,
-				GoroutineCount: runtime.NumGoroutine(),
-				GOMAXPROCS:     runtime.GOMAXPROCS(0),
-				GOGC:           os.Getenv("GOGC"),
-				GODEBUG:        os.Getenv("GODEBUG"),
-			}
-		}
-
 		ins := extpromhttp.NewInstrumentationMiddleware(reg)
 		// TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting.
-		ui.NewQueryUI(logger, reg, stores, webExternalPrefix, webPrefixHeaderName, runtimeInfo, *buildInfo).Register(router, ins)
+		ui.NewQueryUI(logger, reg, stores, webExternalPrefix, webPrefixHeaderName).Register(router, ins)
 
-		api := v1.NewAPI(
+		api := v1.NewQueryAPI(
 			logger,
 			reg,
 			stores,
@@ -462,8 +420,6 @@ func runQuery(
 			flagsMap,
 			instantDefaultMaxSourceResolution,
 			maxConcurrentQueries,
-			runtimeInfo,
-			buildInfo,
 		)
 
 		api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins)

cmd/thanos/rule.go

@@ -32,6 +32,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/thanos-io/thanos/pkg/alert"
+	v1 "github.com/thanos-io/thanos/pkg/api/rule"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/component"
 	"github.com/thanos-io/thanos/pkg/discovery/dns"
@@ -44,7 +45,6 @@ import (
 	"github.com/thanos-io/thanos/pkg/promclient"
 	"github.com/thanos-io/thanos/pkg/query"
 	thanosrules "github.com/thanos-io/thanos/pkg/rules"
-	v1 "github.com/thanos-io/thanos/pkg/rules/api"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	grpcserver "github.com/thanos-io/thanos/pkg/server/grpc"
 	httpserver "github.com/thanos-io/thanos/pkg/server/http"
@@ -170,6 +170,8 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) {
 			return errors.New("--alertmanagers.url and --alertmanagers.config* parameters cannot be defined at the same time")
 		}
 
+		flagsMap := getFlagsMap(cmd.Model().Flags)
+
 		return runRule(g,
 			logger,
 			reg,
@@ -206,6 +208,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) {
 			*dnsSDResolver,
 			comp,
 			*allowOutOfOrderUpload,
+			flagsMap,
 		)
 	}
 }
@@ -293,6 +296,7 @@ func runRule(
 	dnsSDResolver string,
 	comp component.Component,
 	allowOutOfOrderUpload bool,
+	flagsMap map[string]string,
 ) error {
 	metrics := newRuleMetrics(reg)
 
@@ -585,7 +589,7 @@ func runRule(
 		// TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting.
 		ui.NewRuleUI(logger, reg, ruleMgr, alertQueryURL.String(), webExternalPrefix, webPrefixHeaderName).Register(router, ins)
 
-		api := v1.NewAPI(logger, reg, thanosrules.NewGRPCClient(ruleMgr), ruleMgr)
+		api := v1.NewRuleAPI(logger, reg, thanosrules.NewGRPCClient(ruleMgr), ruleMgr, flagsMap)
 		api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins)
 
 		srv := httpserver.New(logger, reg, comp, httpProbe,

cmd/thanos/tools_bucket.go

@@ -24,6 +24,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/route"
 	"github.com/prometheus/prometheus/pkg/labels"
+	v1 "github.com/thanos-io/thanos/pkg/api/blocks"
 	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/compact"
@@ -328,7 +329,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str
 	timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration()
 	label := cmd.Flag("label", "Prometheus label to use as timeline title").String()
 
-	m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error {
+	m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
 		comp := component.Bucket
 		httpProbe := prober.NewHTTP()
 		statusProber := prober.Combine(
@@ -342,9 +343,16 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str
 		)
 
 		router := route.New()
+		ins := extpromhttp.NewInstrumentationMiddleware(reg)
 
 		bucketUI := ui.NewBucketUI(logger, *label, *webExternalPrefix, *webPrefixHeaderName)
-		bucketUI.Register(router, extpromhttp.NewInstrumentationMiddleware(reg))
+		bucketUI.Register(router, ins)
+
+		flagsMap := getFlagsMap(cmd.Model().Flags)
+
+		api := v1.NewBlocksAPI(logger, *label, flagsMap)
+		api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins)
+
 		srv.Handle("/", router)
 
 		if *interval < 5*time.Minute {
@@ -374,7 +382,10 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str
 		if err != nil {
 			return err
 		}
-		fetcher.UpdateOnChange(bucketUI.Set)
+		fetcher.UpdateOnChange(func(blocks []metadata.Meta, err error) {
+			bucketUI.Set(blocks, err)
+			api.Set(blocks, err)
+		})
 
 		ctx, cancel := context.WithCancel(context.Background())
 		g.Add(func() error {