Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: hide docs for thanos clients + restore original cli prefix #14545

Merged
merged 1 commit into from
Oct 21, 2024
Merged

chore: hide docs for thanos clients + restore original cli prefix #14545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 0 additions & 50 deletions docs/sources/shared/configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1789,17 +1789,6 @@ storage:
# CLI flag: -common.storage.congestion-control.hedge.strategy
[strategy: <string> | default = ""]

object_store:
# The gcs_storage_backend block configures the connection to Google Cloud
# Storage object storage backend.
# The CLI flags prefix for this block configuration is: common.storage
[gcs: <gcs_storage_backend>]

# Prefix for all objects stored in the backend storage. For simplicity, it
# may only contain digits and English alphabet letters.
# CLI flag: -common.storage.object-store.storage-prefix
[storage_prefix: <string> | default = ""]

[persist_tokens: <boolean>]

[replication_factor: <int>]
Expand Down Expand Up @@ -2565,35 +2554,6 @@ The `frontend_worker` configures the worker - running within the Loki querier -
[query_scheduler_grpc_client: <grpc_client>]
```

### gcs_storage_backend

The `gcs_storage_backend` block configures the connection to Google Cloud Storage object storage backend.

```yaml
# GCS bucket name
# CLI flag: -<prefix>.object-store.gcs.bucket-name
[bucket_name: <string> | default = ""]

# JSON either from a Google Developers Console client_credentials.json file, or
# a Google Developers service account key. Needs to be valid JSON, not a
# filesystem path. If empty, fallback to Google default logic:
# 1. A JSON file whose path is specified by the GOOGLE_APPLICATION_CREDENTIALS
# environment variable. For workload identity federation, refer to
# https://cloud.google.com/iam/docs/how-to#using-workload-identity-federation on
# how to generate the JSON configuration file for on-prem/non-Google cloud
# platforms.
# 2. A JSON file in a location known to the gcloud command-line tool:
# $HOME/.config/gcloud/application_default_credentials.json.
# 3. On Google Compute Engine it fetches credentials from the metadata server.
# CLI flag: -<prefix>.object-store.gcs.service-account
[service_account: <string> | default = ""]

# The maximum size of the buffer that GCS client for a single PUT request. 0 to
# disable buffering.
# CLI flag: -<prefix>.object-store.gcs.chunk-buffer-size
[chunk_buffer_size: <int> | default = 0]
```

### gcs_storage_config

The `gcs_storage_config` block configures the connection to Google Cloud Storage object storage backend. The supported CLI flags `<prefix>` used to reference this configuration block are:
Expand Down Expand Up @@ -5693,16 +5653,6 @@ congestion_control:
# CLI flag: -use-thanos-objstore
[use_thanos_objstore: <boolean> | default = false]

object_store:
# The gcs_storage_backend block configures the connection to Google Cloud
# Storage object storage backend.
[gcs: <gcs_storage_backend>]

# Prefix for all objects stored in the backend storage. For simplicity, it may
# only contain digits and English alphabet letters.
# CLI flag: -object-store.storage-prefix
[storage_prefix: <string> | default = ""]

# The maximum number of chunks to fetch per batch.
# CLI flag: -store.max-chunk-batch-size
[max_chunk_batch_size: <int> | default = 50]
Expand Down
4 changes: 2 additions & 2 deletions pkg/loki/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ type Storage struct {
Hedging hedging.Config `yaml:"hedging"`
COS ibmcloud.COSConfig `yaml:"cos"`
CongestionControl congestion.Config `yaml:"congestion_control,omitempty"`
ObjectStore bucket.Config `yaml:"object_store"`
ObjectStore bucket.Config `yaml:"object_store" doc:"hidden"`
}

func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
Expand All @@ -94,7 +94,7 @@ func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
s.COS.RegisterFlagsWithPrefix(prefix, f)
s.CongestionControl.RegisterFlagsWithPrefix(prefix, f)

s.ObjectStore.RegisterFlagsWithPrefix(prefix, f)
s.ObjectStore.RegisterFlagsWithPrefix(prefix+"object-store.", f)
}

type FilesystemConfig struct {
Expand Down
20 changes: 9 additions & 11 deletions pkg/storage/bucket/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ var (
// StorageBackendConfig holds configuration for accessing long-term storage.
type StorageBackendConfig struct {
// Backends
S3 s3.Config `yaml:"s3" doc:"hidden"`
S3 s3.Config `yaml:"s3"`
GCS gcs.Config `yaml:"gcs"`
Azure azure.Config `yaml:"azure" doc:"hidden"`
Swift swift.Config `yaml:"swift" doc:"hidden"`
Filesystem filesystem.Config `yaml:"filesystem" doc:"hidden"`
Azure azure.Config `yaml:"azure"`
Swift swift.Config `yaml:"swift"`
Filesystem filesystem.Config `yaml:"filesystem"`

// Used to inject additional backends into the config. Allows for this config to
// be embedded in multiple contexts and support non-object storage based backends.
Expand All @@ -71,14 +71,12 @@ func (cfg *StorageBackendConfig) RegisterFlags(f *flag.FlagSet) {
cfg.RegisterFlagsWithPrefix("", f)
}

func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, _ string, f *flag.FlagSet) {
func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet) {
cfg.GCS.RegisterFlagsWithPrefix(prefix, f)

// TODO: uncomment when other providers are supported
// cfg.S3.RegisterFlagsWithPrefix(prefix, f)
// cfg.Azure.RegisterFlagsWithPrefix(prefix, f)
// cfg.Swift.RegisterFlagsWithPrefix(prefix, f)
// cfg.Filesystem.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f)
cfg.S3.RegisterFlagsWithPrefix(prefix, f)
cfg.Azure.RegisterFlagsWithPrefix(prefix, f)
cfg.Swift.RegisterFlagsWithPrefix(prefix, f)
cfg.Filesystem.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f)
}

func (cfg *StorageBackendConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
Expand Down
6 changes: 3 additions & 3 deletions pkg/storage/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,8 +295,8 @@ type Config struct {
DisableBroadIndexQueries bool `yaml:"disable_broad_index_queries"`
MaxParallelGetChunk int `yaml:"max_parallel_get_chunk"`

UseThanosObjstore bool `yaml:"use_thanos_objstore"`
ObjectStore bucket.Config `yaml:"object_store"`
UseThanosObjstore bool `yaml:"use_thanos_objstore" doc:"hidden`
ObjectStore bucket.Config `yaml:"object_store" doc:"hidden"`

MaxChunkBatchSize int `yaml:"max_chunk_batch_size"`
BoltDBShipperConfig boltdb.IndexCfg `yaml:"boltdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in the form of boltdb files. Required fields only required when boltdb-shipper is defined in config."`
Expand Down Expand Up @@ -326,7 +326,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
cfg.CongestionControl.RegisterFlagsWithPrefix("store.", f)

f.BoolVar(&cfg.UseThanosObjstore, "use-thanos-objstore", false, "Enables the use of thanos-io/objstore clients for connecting to object storage. When set to true, the configuration inside `storage_config.object_store` or `common.storage.object_store` block takes effect.")
cfg.ObjectStore.RegisterFlags(f)
cfg.ObjectStore.RegisterFlagsWithPrefix("object-store.", f)

cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "", f)
f.DurationVar(&cfg.IndexCacheValidity, "store.index-cache-validity", 5*time.Minute, "Cache validity for active index entries. Should be no higher than -ingester.max-chunk-idle.")
Expand Down
Loading