diff --git a/CHANGELOG.md b/CHANGELOG.md index 80cd3cf26a0..02d2cba3634 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,28 @@ * [BUGFIX] Update port spec for GCS docker-compose example [#869](https://github.com/grafana/tempo/pull/869) (@zalegrala) * [BUGFIX] Cortex upgrade to fix an issue where unhealthy compactors can't be forgotten [#878](https://github.com/grafana/tempo/pull/878) (@joe-elliott) * [ENHANCEMENT] Added "query blocks" cli option. [#876](https://github.com/grafana/tempo/pull/876) (@joe-elliott) +* [ENHANCEMENT] Added traceid to `trace too large message`. [#888](https://github.com/grafana/tempo/pull/888) (@mritunjaysharma394) +* [ENHANCEMENT] Add support to tempo workloads to `overrides` from single configmap in microservice mode. [#896](https://github.com/grafana/tempo/pull/896) (@kavirajk) +* [ENHANCEMENT] Make `overrides_config` block name consistent with Loki and Cortex in microservice mode. [#906](https://github.com/grafana/tempo/pull/906) (@kavirajk) +* [ENHANCEMENT] Updated config defaults to reflect better capture operational knowledge. [#913](https://github.com/grafana/tempo/pull/913) (@joe-elliott) + ``` + ingester: + trace_idle_period: 30s => 10s # reduce ingester memory requirements with little impact on querying + flush_check_period: 30s => 10s + query_frontend: + query_shards: 2 => 20 # will massively improve performance on large installs + storage: + trace: + wal: + encoding: none => snappy # snappy has been tested thoroughly and ready for production use + block: + bloom_filter_false_positive: .05 => .01 # will increase total bloom filter size but improve query performance + bloom_filter_shard_size_bytes: 256KiB => 100 KiB # will improve query performance + compactor: + compaction: + chunk_size_bytes: 10 MiB => 5 MiB # will reduce compactor memory needs + compaction_window: 4h => 1h # will allow more compactors to participate in compaction without substantially increasing blocks + ``` * [ENHANCEMENT] Make s3 backend readError logic more robust [#905](https://github.com/grafana/tempo/pull/905) (@wei840222) * [ENHANCEMENT] Include additional detail when searching for traces [#916](https://github.com/grafana/tempo/pull/916) (@zalegrala) * [ENHANCEMENT] Add `gen index` and `gen bloom` commands to tempo-cli. [#903](https://github.com/grafana/tempo/pull/903) (@annanay25) diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md index c451c067947..ae96a0306e8 100644 --- a/docs/tempo/website/configuration/_index.md +++ b/docs/tempo/website/configuration/_index.md @@ -142,10 +142,14 @@ ingester: # number of replicas of each span to make while pushing to the backend replication_factor: 3 - # amount of time before considering a trace complete and flushing it to a block - # (default: 30s) + # amount of time a trace must be idle before flushing it to the wal. + # (default: 10s) [trace_idle_period: ] + # how often to sweep all tenants and move traces from live -> wal -> completed blocks. + # (default: 10s) + [flush_check_period: ] + # maximum size of a block before cutting it # (default: 1073741824 = 1GB) [max_block_bytes: ] @@ -169,7 +173,7 @@ query_frontend: [max_retries: ] # number of shards to split the query into - # (default: 2) + # (default: 20) [query_shards: ] ``` @@ -218,10 +222,10 @@ compactor: # Optional. Duration to keep blocks that have been compacted elsewhere. Default is 1h. [compacted_block_retention: ] - # Optional. Blocks in this time window will be compacted together. Default is 4h. + # Optional. Blocks in this time window will be compacted together. Default is 1h. [compaction_window: ] - # Optional. Amount of data to buffer from input blocks. Default is 10 MB. + # Optional. Amount of data to buffer from input blocks. Default is 5 MiB. [chunk_size_bytes: ] # Optional. Flush data to backend when buffer is this large. Default is 30 MB. @@ -518,19 +522,24 @@ storage: # Example: "wal: /var/tempo/wal" [path: ] - # (experimental) wal encoding/compression. + # wal encoding/compression. # options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd + # (default: snappy) [encoding: ] # block configuration block: # bloom filter false positive rate. lower values create larger filters but fewer false positives - # (default: .05) + # (default: .01) [bloom_filter_false_positive: ] + # maximum size of each bloom filter shard + # (default: 100 KiB) + [bloom_filter_shard_size_bytes: ] + # number of bytes per index record - # (defaultL 1MB) + # (default: 1MiB) [index_downsample_bytes: ] # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd diff --git a/docs/tempo/website/configuration/compression.md b/docs/tempo/website/configuration/compression.md index 8b109ffab48..a587c740165 100644 --- a/docs/tempo/website/configuration/compression.md +++ b/docs/tempo/website/configuration/compression.md @@ -34,16 +34,15 @@ It is important to note that although all of these compression formats are suppo we use zstd and it's possible/probable that the other compression algorithms may have issue at scale. Please file an issue if you stumble upon any problems! -## WAL (Experimental) +## WAL -The WAL also supports compression. By default this is turned off because it comes with a small performance penalty. -However, it does reduce disk i/o and adds checksums to the WAL which are valuable in higher volume installations. +The WAL also supports compression. By default this is configured to use snappy. This comes with a small performance +penalty but reduces disk I/O and and adds checksums to the WAL. All of the above configuration options are supported +but only snappy has been tested at scale. ``` storage: trace: wal: - encoding: none -``` - -If WAL compression is turned on it is recommend to use snappy. All of the above options are supported. \ No newline at end of file + encoding: snappy +``` \ No newline at end of file diff --git a/docs/tempo/website/configuration/manifest.md b/docs/tempo/website/configuration/manifest.md index 38da1e545fa..5bf957f711b 100644 --- a/docs/tempo/website/configuration/manifest.md +++ b/docs/tempo/website/configuration/manifest.md @@ -16,7 +16,7 @@ go run ./cmd/tempo --storage.trace.backend=local --storage.trace.local.path=/tmp ## Complete Configuration -> **Note**: This manifest was generated on 7th of June 2021. +> **Note**: This manifest was generated on 2021-08-25. ```yaml target: all @@ -181,7 +181,7 @@ query_frontend: port: 0 downstream_url: "" max_retries: 2 - query_shards: 2 + query_shards: 20 compactor: ring: kvstore: @@ -223,9 +223,9 @@ compactor: instance_addr: "" wait_active_instance_timeout: 10m0s compaction: - chunk_size_bytes: 10485760 + chunk_size_bytes: 5242880 flush_size_bytes: 31457280 - compaction_window: 4h0m0s + compaction_window: 1h0m0s max_compaction_objects: 6000000 max_block_bytes: 107374182400 block_retention: 336h0m0s @@ -282,9 +282,9 @@ ingester: port: 0 id: hostname concurrent_flushes: 16 - flush_check_period: 30s + flush_check_period: 10s flush_op_timeout: 5m0s - trace_idle_period: 30s + trace_idle_period: 10s max_block_duration: 1h0m0s max_block_bytes: 1073741824 complete_block_timeout: 15m0s @@ -298,12 +298,12 @@ storage: path: /tmp/tempo/wal completedfilepath: /tmp/tempo/wal/completed blocksfilepath: /tmp/tempo/wal/blocks - encoding: none + encoding: snappy block: index_downsample_bytes: 1048576 index_page_size_bytes: 256000 - bloom_filter_false_positive: 0.05 - bloom_filter_shard_size_bytes: 256000 + bloom_filter_false_positive: 0.01 + bloom_filter_shard_size_bytes: 102400 encoding: zstd blocklist_poll: 5m0s blocklist_poll_concurrency: 50 diff --git a/example/docker-compose/azure/tempo-azure.yaml b/example/docker-compose/azure/tempo-azure.yaml index 19a59910650..b5d262dd009 100644 --- a/example/docker-compose/azure/tempo-azure.yaml +++ b/example/docker-compose/azure/tempo-azure.yaml @@ -37,7 +37,7 @@ storage: encoding: zstd # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd wal: path: /tmp/tempo/wal # where to store the the wal locally - encoding: none # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd + encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd azure: container-name: tempo # how to store data in azure endpoint-suffix: azurite:10000 diff --git a/example/docker-compose/distributed/tempo-distributed.yaml b/example/docker-compose/distributed/tempo-distributed.yaml index e29dafd479e..30fd0f2111c 100644 --- a/example/docker-compose/distributed/tempo-distributed.yaml +++ b/example/docker-compose/distributed/tempo-distributed.yaml @@ -55,7 +55,7 @@ storage: insecure: true wal: path: /tmp/tempo/wal # where to store the the wal locally - encoding: none # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd + encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd local: path: /tmp/tempo/blocks pool: diff --git a/example/docker-compose/gcs/tempo-gcs.yaml b/example/docker-compose/gcs/tempo-gcs.yaml index 1fdb90a2394..303be04a97c 100644 --- a/example/docker-compose/gcs/tempo-gcs.yaml +++ b/example/docker-compose/gcs/tempo-gcs.yaml @@ -38,7 +38,7 @@ storage: encoding: zstd # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd wal: path: /tmp/tempo/wal # where to store the the wal locally - encoding: none # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd + encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd gcs: bucket_name: tempo endpoint: https://gcs:4443/storage/v1/ diff --git a/example/docker-compose/local/tempo-local.yaml b/example/docker-compose/local/tempo-local.yaml index 1466cfee495..9667b5d8367 100644 --- a/example/docker-compose/local/tempo-local.yaml +++ b/example/docker-compose/local/tempo-local.yaml @@ -37,7 +37,7 @@ storage: encoding: zstd # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd wal: path: /tmp/tempo/wal # where to store the the wal locally - encoding: none # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd + encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd local: path: /tmp/tempo/blocks pool: diff --git a/example/docker-compose/s3/tempo-s3.yaml b/example/docker-compose/s3/tempo-s3.yaml index 6cb1e916f2f..7bd6272d051 100644 --- a/example/docker-compose/s3/tempo-s3.yaml +++ b/example/docker-compose/s3/tempo-s3.yaml @@ -38,7 +38,7 @@ storage: encoding: zstd # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd wal: path: /tmp/tempo/wal # where to store the the wal locally - encoding: none # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd + encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd s3: bucket: tempo # how to store data in s3 endpoint: minio:9000 diff --git a/modules/compactor/config.go b/modules/compactor/config.go index 18c3b6c95aa..236da05c446 100644 --- a/modules/compactor/config.go +++ b/modules/compactor/config.go @@ -20,7 +20,7 @@ type Config struct { // RegisterFlagsAndApplyDefaults registers the flags. func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) { cfg.Compactor = tempodb.CompactorConfig{ - ChunkSizeBytes: 10 * 1024 * 1024, // 10 MiB + ChunkSizeBytes: 5 * 1024 * 1024, // 5 MiB FlushSizeBytes: tempodb.DefaultFlushSizeBytes, CompactedBlockRetention: time.Hour, RetentionConcurrency: tempodb.DefaultRetentionConcurrency, @@ -33,6 +33,6 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) f.DurationVar(&cfg.Compactor.BlockRetention, util.PrefixConfig(prefix, "compaction.block-retention"), 14*24*time.Hour, "Duration to keep blocks/traces.") f.IntVar(&cfg.Compactor.MaxCompactionObjects, util.PrefixConfig(prefix, "compaction.max-objects-per-block"), 6000000, "Maximum number of traces in a compacted block.") f.Uint64Var(&cfg.Compactor.MaxBlockBytes, util.PrefixConfig(prefix, "compaction.max-block-bytes"), 100*1024*1024*1024 /* 100GB */, "Maximum size of a compacted block.") - f.DurationVar(&cfg.Compactor.MaxCompactionRange, util.PrefixConfig(prefix, "compaction.compaction-window"), 4*time.Hour, "Maximum time window across which to compact blocks.") + f.DurationVar(&cfg.Compactor.MaxCompactionRange, util.PrefixConfig(prefix, "compaction.compaction-window"), time.Hour, "Maximum time window across which to compact blocks.") cfg.OverrideRingKey = ring.CompactorRingKey } diff --git a/modules/frontend/config.go b/modules/frontend/config.go index d3239703596..31624fd945e 100644 --- a/modules/frontend/config.go +++ b/modules/frontend/config.go @@ -18,7 +18,7 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.Config.Handler.LogQueriesLongerThan = 0 cfg.Config.FrontendV1.MaxOutstandingPerTenant = 100 cfg.MaxRetries = 2 - cfg.QueryShards = 2 + cfg.QueryShards = 20 } type CortexNoQuerierLimits struct{} diff --git a/modules/ingester/config.go b/modules/ingester/config.go index 315ba99558f..f385244d0e4 100644 --- a/modules/ingester/config.go +++ b/modules/ingester/config.go @@ -35,10 +35,10 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.LifecyclerConfig.RingConfig.HeartbeatTimeout = 5 * time.Minute cfg.ConcurrentFlushes = 16 - cfg.FlushCheckPeriod = 30 * time.Second + cfg.FlushCheckPeriod = 10 * time.Second cfg.FlushOpTimeout = 5 * time.Minute - f.DurationVar(&cfg.MaxTraceIdle, prefix+".trace-idle-period", 30*time.Second, "Duration after which to consider a trace complete if no spans have been received") + f.DurationVar(&cfg.MaxTraceIdle, prefix+".trace-idle-period", 10*time.Second, "Duration after which to consider a trace complete if no spans have been received") f.DurationVar(&cfg.MaxBlockDuration, prefix+".max-block-duration", time.Hour, "Maximum duration which the head block can be appended to before cutting it.") f.Uint64Var(&cfg.MaxBlockBytes, prefix+".max-block-bytes", 1024*1024*1024, "Maximum size of the head block before cutting it.") f.DurationVar(&cfg.CompleteBlockTimeout, prefix+".complete-block-timeout", 3*tempodb.DefaultBlocklistPoll, "Duration to keep head blocks in the ingester after they have been cut.") diff --git a/modules/storage/config.go b/modules/storage/config.go index 4100f3bb00d..72ee7fad6c7 100644 --- a/modules/storage/config.go +++ b/modules/storage/config.go @@ -34,11 +34,11 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.Trace.WAL = &wal.Config{} f.StringVar(&cfg.Trace.WAL.Filepath, util.PrefixConfig(prefix, "trace.wal.path"), "/var/tempo/wal", "Path at which store WAL blocks.") - cfg.Trace.WAL.Encoding = backend.EncNone + cfg.Trace.WAL.Encoding = backend.EncSnappy cfg.Trace.Block = &encoding.BlockConfig{} - f.Float64Var(&cfg.Trace.Block.BloomFP, util.PrefixConfig(prefix, "trace.block.bloom-filter-false-positive"), .05, "Bloom Filter False Positive.") - f.IntVar(&cfg.Trace.Block.BloomShardSizeBytes, util.PrefixConfig(prefix, "trace.block.bloom-filter-shard-size-bytes"), 250*1024, "Bloom Filter Shard Size in bytes.") + f.Float64Var(&cfg.Trace.Block.BloomFP, util.PrefixConfig(prefix, "trace.block.bloom-filter-false-positive"), .01, "Bloom Filter False Positive.") + f.IntVar(&cfg.Trace.Block.BloomShardSizeBytes, util.PrefixConfig(prefix, "trace.block.bloom-filter-shard-size-bytes"), 100*1024, "Bloom Filter Shard Size in bytes.") f.IntVar(&cfg.Trace.Block.IndexDownsampleBytes, util.PrefixConfig(prefix, "trace.block.index-downsample-bytes"), 1024*1024, "Number of bytes (before compression) per index record.") f.IntVar(&cfg.Trace.Block.IndexPageSizeBytes, util.PrefixConfig(prefix, "trace.block.index-page-size-bytes"), 250*1024, "Number of bytes per index page.") cfg.Trace.Block.Encoding = backend.EncZstd