[CHORE] adding thanos upload-snapshot command (#6884)

Signed-off-by: Nicolas Takashi <nicolas.tcs@hotmail.com>

CHANGELOG.md

@@ -29,6 +29,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#6984](https://github.com/thanos-io/thanos/pull/6984) Store Gateway: Added `--store.index-header-lazy-download-strategy` to specify how to lazily download index headers when lazy mmap is enabled.
 - [#6887](https://github.com/thanos-io/thanos/pull/6887) Query Frontend: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing custom dashboard queries to be incorrect due to the added label.
 - [#7028](https://github.com/thanos-io/thanos/pull/7028) Query|Query Frontend: Add new `--query-frontend.enable-x-functions` flag to enable experimental extended functions.
+- [#6884](https://github.com/thanos-io/thanos/pull/6884) Tools: Add upload-block command to upload blocks to object storage.
 
 ### Changed
 

cmd/thanos/tools_bucket.go

@@ -46,6 +46,7 @@ import (
 	v1 "github.com/thanos-io/thanos/pkg/api/blocks"
 	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
+	"github.com/thanos-io/thanos/pkg/clientconfig"
 	"github.com/thanos-io/thanos/pkg/compact"
 	"github.com/thanos-io/thanos/pkg/compact/downsample"
 	"github.com/thanos-io/thanos/pkg/compactv2"
@@ -56,9 +57,11 @@ import (
 	"github.com/thanos-io/thanos/pkg/logging"
 	"github.com/thanos-io/thanos/pkg/model"
 	"github.com/thanos-io/thanos/pkg/prober"
+	"github.com/thanos-io/thanos/pkg/promclient"
 	"github.com/thanos-io/thanos/pkg/replicate"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	httpserver "github.com/thanos-io/thanos/pkg/server/http"
+	"github.com/thanos-io/thanos/pkg/shipper"
 	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/ui"
 	"github.com/thanos-io/thanos/pkg/verifier"
@@ -158,6 +161,11 @@ type bucketMarkBlockConfig struct {
 	removeMarker bool
 }
 
+type bucketUploadBlocksConfig struct {
+	path       string
+	prometheus prometheusConfig
+}
+
 func (tbc *bucketVerifyConfig) registerBucketVerifyFlag(cmd extkingpin.FlagClause) *bucketVerifyConfig {
 	cmd.Flag("repair", "Attempt to repair blocks for which issues were detected").
 		Short('r').Default("false").BoolVar(&tbc.repair)
@@ -277,6 +285,30 @@ func (tbc *bucketRetentionConfig) registerBucketRetentionFlag(cmd extkingpin.Fla
 	return tbc
 }
 
+func (tbc *bucketUploadBlocksConfig) registerBucketUploadBlocksFlag(cmd extkingpin.FlagClause) *bucketUploadBlocksConfig {
+	cmd.Flag("path", "Path to the directory containing blocks to upload.").Default("./data").StringVar(&tbc.path)
+
+	cmd.Flag("prometheus.url",
+		"URL at which to reach Prometheus's API. For better performance use local network.").
+		Default("http://localhost:9090").URLVar(&tbc.prometheus.url)
+	cmd.Flag("prometheus.ready_timeout",
+		"Maximum time to wait for the Prometheus instance to start up").
+		Default("10m").DurationVar(&tbc.prometheus.readyTimeout)
+	cmd.Flag("prometheus.get_config_interval",
+		"How often to get Prometheus config").
+		Default("30s").DurationVar(&tbc.prometheus.getConfigInterval)
+	cmd.Flag("prometheus.get_config_timeout",
+		"Timeout for getting Prometheus config").
+		Default("5s").DurationVar(&tbc.prometheus.getConfigTimeout)
+	tbc.prometheus.httpClient = extflag.RegisterPathOrContent(
+		cmd,
+		"prometheus.http-client",
+		"YAML file or string with http client configs. See Format details: https://thanos.io/tip/components/sidecar.md/#configuration.",
+	)
+
+	return tbc
+}
+
 func registerBucket(app extkingpin.AppClause) {
 	cmd := app.Command("bucket", "Bucket utility commands")
 
@@ -291,6 +323,7 @@ func registerBucket(app extkingpin.AppClause) {
 	registerBucketMarkBlock(cmd, objStoreConfig)
 	registerBucketRewrite(cmd, objStoreConfig)
 	registerBucketRetention(cmd, objStoreConfig)
+	registerBucketUploadBlocks(cmd, objStoreConfig)
 }
 
 func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.PathOrContent) {
@@ -1420,3 +1453,91 @@ func registerBucketRetention(app extkingpin.AppClause, objStoreConfig *extflag.P
 		return nil
 	})
 }
+
+func registerBucketUploadBlocks(app extkingpin.AppClause, objStoreConfig *extflag.PathOrContent) {
+	cmd := app.Command("upload-blocks", "Upload blocks push blocks from the provided path to the object storage.")
+
+	tbc := &bucketUploadBlocksConfig{}
+	tbc.registerBucketUploadBlocksFlag(cmd)
+
+	cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error {
+		ctx := context.Background()
+		httpConfContentYaml, err := tbc.prometheus.httpClient.Content()
+		if err != nil {
+			return errors.Wrap(err, "getting http client config")
+		}
+
+		httpClientConfig, err := clientconfig.NewHTTPClientConfigFromYAML(httpConfContentYaml)
+		if err != nil {
+			return errors.Wrap(err, "parsing http config YAML")
+		}
+
+		httpClient, err := clientconfig.NewHTTPClient(*httpClientConfig, "thanos-tool")
+		if err != nil {
+			return errors.Wrap(err, "Improper http client config")
+		}
+
+		m := &promMetadata{
+			promURL: tbc.prometheus.url,
+			client:  promclient.NewWithTracingClient(logger, httpClient, "thanos-tool"),
+		}
+
+		err = runutil.Retry(2*time.Second, ctx.Done(), func() error {
+			if err := m.UpdateLabels(ctx); err != nil {
+				level.Warn(logger).Log(
+					"msg", "failed to fetch initial external labels. Is Prometheus running? Retrying",
+					"err", err,
+				)
+				return err
+			}
+
+			level.Info(logger).Log(
+				"msg", "successfully loaded prometheus external labels",
+				"external_labels", m.Labels().String(),
+			)
+			return nil
+		})
+
+		if err != nil {
+			return errors.Wrap(err, "initial external labels query")
+		}
+
+		if len(m.Labels()) == 0 {
+			return errors.New("no external labels configured on Prometheus server, uniquely identifying external labels must be configured; see https://thanos.io/tip/thanos/storage.md#external-labels for details.")
+		}
+
+		confContentYaml, err := objStoreConfig.Content()
+		if err != nil {
+			return err
+		}
+
+		bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String())
+		if err != nil {
+			return err
+		}
+
+		bkt = objstoretracing.WrapWithTraces(objstore.WrapWithMetrics(bkt, extprom.WrapRegistererWithPrefix("thanos_", reg), bkt.Name()))
+
+		// Ensure we close up everything properly.
+		defer func() {
+			if err != nil {
+				runutil.CloseWithLogOnErr(logger, bkt, "bucket client")
+			}
+		}()
+
+		if err := promclient.IsDirAccessible(tbc.path); err != nil {
+			level.Error(logger).Log("err", err)
+		}
+
+		defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client")
+
+		s := shipper.New(logger, reg, tbc.path, bkt, m.Labels, metadata.BucketUploadSource,
+			nil, false, metadata.HashFunc(""), shipper.DefaultMetaFilename)
+
+		if _, err := s.Sync(ctx); err != nil {
+			level.Error(logger).Log("msg", "shipper sync", "err", err)
+		}
+
+		return nil
+	})
+}

docs/components/tools.md

@@ -78,6 +78,9 @@ Subcommands:
     Retention applies retention policies on the given bucket. Please make sure
     no compactor is running on the same bucket at the same time.
 
+  tools bucket upload-blocks [<flags>]
+    Upload blocks push blocks from the provided path to the object storeage.
+
   tools rules-check --rules=RULES
     Check if the rule files are valid or not.
 
@@ -187,6 +190,9 @@ Subcommands:
     Retention applies retention policies on the given bucket. Please make sure
     no compactor is running on the same bucket at the same time.
 
+  tools bucket upload-blocks [<flags>]
+    Upload blocks push blocks from the provided path to the object storeage.
+
 
 ```
 
@@ -841,6 +847,62 @@ Flags:
 
 ```
 
+### Bucket Upload Blocks
+
+`tools bucket upload-blocks` uploads a blocks created on the given bucket.
+
+```$ mdox-exec="thanos tools bucket upload-blocks --help"
+usage: thanos tools bucket upload-blocks [<flags>]
+
+Upload blocks push blocks from the provided path to the object storeage.
+
+Flags:
+  -h, --help               Show context-sensitive help (also try --help-long and
+                           --help-man).
+      --log.format=logfmt  Log format to use. Possible options: logfmt or json.
+      --log.level=info     Log filtering level.
+      --objstore.config=<content>
+                           Alternative to 'objstore.config-file' flag (mutually
+                           exclusive). Content of YAML file that contains
+                           object store configuration. See format details:
+                           https://thanos.io/tip/thanos/storage.md/#configuration
+      --objstore.config-file=<file-path>
+                           Path to YAML file that contains object
+                           store configuration. See format details:
+                           https://thanos.io/tip/thanos/storage.md/#configuration
+      --path="./data"      Path to the directory containing blocks to upload.
+      --prometheus.get_config_interval=30s
+                           How often to get Prometheus config
+      --prometheus.get_config_timeout=5s
+                           Timeout for getting Prometheus config
+      --prometheus.http-client=<content>
+                           Alternative to 'prometheus.http-client-file' flag
+                           (mutually exclusive). Content of YAML file or string
+                           with http client configs. See Format details:
+                           https://thanos.io/tip/components/sidecar.md/#configuration.
+      --prometheus.http-client-file=<file-path>
+                           Path to YAML file or string with http
+                           client configs. See Format details:
+                           https://thanos.io/tip/components/sidecar.md/#configuration.
+      --prometheus.ready_timeout=10m
+                           Maximum time to wait for the Prometheus instance to
+                           start up
+      --prometheus.url=http://localhost:9090
+                           URL at which to reach Prometheus's API. For better
+                           performance use local network.
+      --tracing.config=<content>
+                           Alternative to 'tracing.config-file' flag
+                           (mutually exclusive). Content of YAML file
+                           with tracing configuration. See format details:
+                           https://thanos.io/tip/thanos/tracing.md/#configuration
+      --tracing.config-file=<file-path>
+                           Path to YAML file with tracing
+                           configuration. See format details:
+                           https://thanos.io/tip/thanos/tracing.md/#configuration
+      --version            Show application version.
+
+```
+
 ## Rules-check
 
 The `tools rules-check` subcommand contains tools for validation of Prometheus rules.

pkg/block/metadata/meta.go

@@ -41,6 +41,7 @@ const (
 	RulerSource           SourceType = "ruler"
 	BucketRepairSource    SourceType = "bucket.repair"
 	BucketRewriteSource   SourceType = "bucket.rewrite"
+	BucketUploadSource    SourceType = "bucket.upload"
 	TestSource            SourceType = "test"
 )
 

pkg/component/component.go

@@ -111,6 +111,7 @@ var (
 	Bucket          = source{component: component{name: "bucket"}}
 	Cleanup         = source{component: component{name: "cleanup"}}
 	Mark            = source{component: component{name: "mark"}}
+	Upload          = source{component: component{name: "upload"}}
 	Rewrite         = source{component: component{name: "rewrite"}}
 	Retention       = source{component: component{name: "retention"}}
 	Compact         = source{component: component{name: "compact"}}

pkg/promclient/promclient.go

@@ -161,6 +161,22 @@ func IsWALDirAccessible(dir string) error {
 	return nil
 }
 
+// IsDirAccessible returns no error if dir can be found.
+func IsDirAccessible(dir string) error {
+	const errMsg = "Dir is not accessible."
+
+	f, err := os.Stat(dir)
+	if err != nil {
+		return errors.Wrap(err, errMsg)
+	}
+
+	if !f.IsDir() {
+		return errors.New(errMsg)
+	}
+
+	return nil
+}
+
 // ExternalLabels returns sorted external labels from /api/v1/status/config Prometheus endpoint.
 // Note that configuration can be hot reloadable on Prometheus, so this config might change in runtime.
 func (c *Client) ExternalLabels(ctx context.Context, base *url.URL) (labels.Labels, error) {