diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index 001bbc8409e4..6ca10719d577 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -1789,17 +1789,6 @@ storage: # CLI flag: -common.storage.congestion-control.hedge.strategy [strategy: | default = ""] - object_store: - # The gcs_storage_backend block configures the connection to Google Cloud - # Storage object storage backend. - # The CLI flags prefix for this block configuration is: common.storage - [gcs: ] - - # Prefix for all objects stored in the backend storage. For simplicity, it - # may only contain digits and English alphabet letters. - # CLI flag: -common.storage.object-store.storage-prefix - [storage_prefix: | default = ""] - [persist_tokens: ] [replication_factor: ] @@ -2565,35 +2554,6 @@ The `frontend_worker` configures the worker - running within the Loki querier - [query_scheduler_grpc_client: ] ``` -### gcs_storage_backend - -The `gcs_storage_backend` block configures the connection to Google Cloud Storage object storage backend. - -```yaml -# GCS bucket name -# CLI flag: -.object-store.gcs.bucket-name -[bucket_name: | default = ""] - -# JSON either from a Google Developers Console client_credentials.json file, or -# a Google Developers service account key. Needs to be valid JSON, not a -# filesystem path. If empty, fallback to Google default logic: -# 1. A JSON file whose path is specified by the GOOGLE_APPLICATION_CREDENTIALS -# environment variable. For workload identity federation, refer to -# https://cloud.google.com/iam/docs/how-to#using-workload-identity-federation on -# how to generate the JSON configuration file for on-prem/non-Google cloud -# platforms. -# 2. A JSON file in a location known to the gcloud command-line tool: -# $HOME/.config/gcloud/application_default_credentials.json. -# 3. On Google Compute Engine it fetches credentials from the metadata server. -# CLI flag: -.object-store.gcs.service-account -[service_account: | default = ""] - -# The maximum size of the buffer that GCS client for a single PUT request. 0 to -# disable buffering. -# CLI flag: -.object-store.gcs.chunk-buffer-size -[chunk_buffer_size: | default = 0] -``` - ### gcs_storage_config The `gcs_storage_config` block configures the connection to Google Cloud Storage object storage backend. The supported CLI flags `` used to reference this configuration block are: @@ -5693,16 +5653,6 @@ congestion_control: # CLI flag: -use-thanos-objstore [use_thanos_objstore: | default = false] -object_store: - # The gcs_storage_backend block configures the connection to Google Cloud - # Storage object storage backend. - [gcs: ] - - # Prefix for all objects stored in the backend storage. For simplicity, it may - # only contain digits and English alphabet letters. - # CLI flag: -object-store.storage-prefix - [storage_prefix: | default = ""] - # The maximum number of chunks to fetch per batch. # CLI flag: -store.max-chunk-batch-size [max_chunk_batch_size: | default = 50] diff --git a/pkg/loki/common/common.go b/pkg/loki/common/common.go index 9fef8c72635d..f1a3c58055d0 100644 --- a/pkg/loki/common/common.go +++ b/pkg/loki/common/common.go @@ -79,7 +79,7 @@ type Storage struct { Hedging hedging.Config `yaml:"hedging"` COS ibmcloud.COSConfig `yaml:"cos"` CongestionControl congestion.Config `yaml:"congestion_control,omitempty"` - ObjectStore bucket.Config `yaml:"object_store"` + ObjectStore bucket.Config `yaml:"object_store" doc:"hidden"` } func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { @@ -94,7 +94,7 @@ func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { s.COS.RegisterFlagsWithPrefix(prefix, f) s.CongestionControl.RegisterFlagsWithPrefix(prefix, f) - s.ObjectStore.RegisterFlagsWithPrefix(prefix, f) + s.ObjectStore.RegisterFlagsWithPrefix(prefix+"object-store.", f) } type FilesystemConfig struct { diff --git a/pkg/storage/bucket/client.go b/pkg/storage/bucket/client.go index 94433dbd4a76..026f3e3957f3 100644 --- a/pkg/storage/bucket/client.go +++ b/pkg/storage/bucket/client.go @@ -50,11 +50,11 @@ var ( // StorageBackendConfig holds configuration for accessing long-term storage. type StorageBackendConfig struct { // Backends - S3 s3.Config `yaml:"s3" doc:"hidden"` + S3 s3.Config `yaml:"s3"` GCS gcs.Config `yaml:"gcs"` - Azure azure.Config `yaml:"azure" doc:"hidden"` - Swift swift.Config `yaml:"swift" doc:"hidden"` - Filesystem filesystem.Config `yaml:"filesystem" doc:"hidden"` + Azure azure.Config `yaml:"azure"` + Swift swift.Config `yaml:"swift"` + Filesystem filesystem.Config `yaml:"filesystem"` // Used to inject additional backends into the config. Allows for this config to // be embedded in multiple contexts and support non-object storage based backends. @@ -71,14 +71,12 @@ func (cfg *StorageBackendConfig) RegisterFlags(f *flag.FlagSet) { cfg.RegisterFlagsWithPrefix("", f) } -func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, _ string, f *flag.FlagSet) { +func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet) { cfg.GCS.RegisterFlagsWithPrefix(prefix, f) - - // TODO: uncomment when other providers are supported - // cfg.S3.RegisterFlagsWithPrefix(prefix, f) - // cfg.Azure.RegisterFlagsWithPrefix(prefix, f) - // cfg.Swift.RegisterFlagsWithPrefix(prefix, f) - // cfg.Filesystem.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f) + cfg.S3.RegisterFlagsWithPrefix(prefix, f) + cfg.Azure.RegisterFlagsWithPrefix(prefix, f) + cfg.Swift.RegisterFlagsWithPrefix(prefix, f) + cfg.Filesystem.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f) } func (cfg *StorageBackendConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go index ac6964e4d4db..e0cc1ff0e738 100644 --- a/pkg/storage/factory.go +++ b/pkg/storage/factory.go @@ -295,8 +295,8 @@ type Config struct { DisableBroadIndexQueries bool `yaml:"disable_broad_index_queries"` MaxParallelGetChunk int `yaml:"max_parallel_get_chunk"` - UseThanosObjstore bool `yaml:"use_thanos_objstore"` - ObjectStore bucket.Config `yaml:"object_store"` + UseThanosObjstore bool `yaml:"use_thanos_objstore" doc:"hidden` + ObjectStore bucket.Config `yaml:"object_store" doc:"hidden"` MaxChunkBatchSize int `yaml:"max_chunk_batch_size"` BoltDBShipperConfig boltdb.IndexCfg `yaml:"boltdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in the form of boltdb files. Required fields only required when boltdb-shipper is defined in config."` @@ -326,7 +326,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.CongestionControl.RegisterFlagsWithPrefix("store.", f) f.BoolVar(&cfg.UseThanosObjstore, "use-thanos-objstore", false, "Enables the use of thanos-io/objstore clients for connecting to object storage. When set to true, the configuration inside `storage_config.object_store` or `common.storage.object_store` block takes effect.") - cfg.ObjectStore.RegisterFlags(f) + cfg.ObjectStore.RegisterFlagsWithPrefix("object-store.", f) cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "", f) f.DurationVar(&cfg.IndexCacheValidity, "store.index-cache-validity", 5*time.Minute, "Cache validity for active index entries. Should be no higher than -ingester.max-chunk-idle.")