From 4cb449af6670041f42cf6962a6bbe0ef77abc5dd Mon Sep 17 00:00:00 2001 From: szalai1 Date: Sun, 17 Mar 2019 22:17:00 +0000 Subject: [PATCH 1/4] add trace functionality to S3 client `minio.Client` has a `TraceOn` method which will be called when one set the `traceon: true` in the bucket config. This was a feature request here https://github.com/improbable-eng/thanos/issues/530 --- docs/storage.md | 2 ++ pkg/objstore/s3/s3.go | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/docs/storage.md b/docs/storage.md index 1962ba3472..b34d669fc3 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -48,6 +48,7 @@ config: encrypt_sse: false secret_key: "" put_user_metadata: {} + traceon: false http_config: idle_conn_timeout: 0s insecure_skip_verify: false @@ -62,6 +63,7 @@ For debug and testing purposes you can set * `insecure: true` to switch to plain insecure HTTP instead of HTTPS * `http_config.insecure_skip_verify: true` to disable TLS certificate verification (if your S3 based storage is using a self-signed certificate, for example) +* `traceon: true` to enable the mino client's verbose logging. Each request and response will be logged into the debug logger, so debug level logging must be enabled for this functionality. ### Credentials By default Thanos will try to retrieve credentials from the following sources: diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 2bfc1041ee..70f71c7c26 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -44,6 +44,7 @@ type Config struct { SecretKey string `yaml:"secret_key"` PutUserMetadata map[string]string `yaml:"put_user_metadata"` HTTPConfig HTTPConfig `yaml:"http_config"` + TraceOn bool `yaml:"traceon"` } // HTTPConfig stores the http.Transport configuration for the s3 minio client. @@ -152,6 +153,11 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B sse = encrypt.NewSSE() } + if config.TraceOn { + logWriter := log.NewStdlibAdapter(level.Debug(logger), log.MessageKey("s3TraceMsg")) + client.TraceOn(logWriter) + } + bkt := &Bucket{ logger: logger, name: config.Bucket, From 86c0757bcdc1fc0afc852bdf4b1a55b7dbc81101 Mon Sep 17 00:00:00 2001 From: szalai1 Date: Mon, 18 Mar 2019 22:31:15 +0000 Subject: [PATCH 2/4] typo --- docs/storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage.md b/docs/storage.md index b34d669fc3..7444ed459d 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -63,7 +63,7 @@ For debug and testing purposes you can set * `insecure: true` to switch to plain insecure HTTP instead of HTTPS * `http_config.insecure_skip_verify: true` to disable TLS certificate verification (if your S3 based storage is using a self-signed certificate, for example) -* `traceon: true` to enable the mino client's verbose logging. Each request and response will be logged into the debug logger, so debug level logging must be enabled for this functionality. +* `traceon: true` to enable the minio client's verbose logging. Each request and response will be logged into the debug logger, so debug level logging must be enabled for this functionality. ### Credentials By default Thanos will try to retrieve credentials from the following sources: From 12b71fe37a584e170b5a5e9c1917df97a7e2c28a Mon Sep 17 00:00:00 2001 From: szalai1 Date: Tue, 19 Mar 2019 07:55:25 +0000 Subject: [PATCH 3/4] make docs --- docs/storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage.md b/docs/storage.md index 7444ed459d..9949bc11ef 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -48,10 +48,10 @@ config: encrypt_sse: false secret_key: "" put_user_metadata: {} - traceon: false http_config: idle_conn_timeout: 0s insecure_skip_verify: false + traceon: false ``` AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) From 46e68c1a2c9b87c54620e972a7ef84ed769b1e9f Mon Sep 17 00:00:00 2001 From: szalai1 Date: Thu, 21 Mar 2019 20:45:23 +0000 Subject: [PATCH 4/4] change s3 trace configuration --- docs/storage.md | 5 +++-- pkg/objstore/s3/s3.go | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index 9949bc11ef..b074090763 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -51,7 +51,8 @@ config: http_config: idle_conn_timeout: 0s insecure_skip_verify: false - traceon: false + trace: + enable: false ``` AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) @@ -63,7 +64,7 @@ For debug and testing purposes you can set * `insecure: true` to switch to plain insecure HTTP instead of HTTPS * `http_config.insecure_skip_verify: true` to disable TLS certificate verification (if your S3 based storage is using a self-signed certificate, for example) -* `traceon: true` to enable the minio client's verbose logging. Each request and response will be logged into the debug logger, so debug level logging must be enabled for this functionality. +* `trace.enable: true` to enable the minio client's verbose logging. Each request and response will be logged into the debug logger, so debug level logging must be enabled for this functionality. ### Credentials By default Thanos will try to retrieve credentials from the following sources: diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 70f71c7c26..c0fe3b98d9 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -44,7 +44,11 @@ type Config struct { SecretKey string `yaml:"secret_key"` PutUserMetadata map[string]string `yaml:"put_user_metadata"` HTTPConfig HTTPConfig `yaml:"http_config"` - TraceOn bool `yaml:"traceon"` + TraceConfig TraceConfig `yaml:"trace"` +} + +type TraceConfig struct { + Enable bool `yaml:"enable"` } // HTTPConfig stores the http.Transport configuration for the s3 minio client. @@ -153,7 +157,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B sse = encrypt.NewSSE() } - if config.TraceOn { + if config.TraceConfig.Enable { logWriter := log.NewStdlibAdapter(level.Debug(logger), log.MessageKey("s3TraceMsg")) client.TraceOn(logWriter) }