Skip to content

Commit

Permalink
Remove support schema flags, only use config file. (grafana#2221)
Browse files Browse the repository at this point in the history 
* Remove support schema flags, only use config file.

Also rename the schema config file flag to something sane.

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

* Update docs to use the schema file.

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

* Deprecate not remove the config flag.

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

* Make integration tests pass?

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

* Remove support schema flags, only use config file.

Also rename the schema config file flag to something sane.

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

* Update docs to use the schema file.

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

* Deprecate not remove the config flag.

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

* Update docs/configuration/schema-config-reference.md

Signed-off-by: Marco Pracucci <marco@pracucci.com>

* Fixed schema config doc

Signed-off-by: Marco Pracucci <marco@pracucci.com>

* Fixes after rebase

Signed-off-by: Marco Pracucci <marco@pracucci.com>

* Remove duplicated entry from CHANGELOG

Signed-off-by: Marco Pracucci <marco@pracucci.com>

Co-authored-by: Marco Pracucci <marco@pracucci.com>
  • Loading branch information
@gouthamve @pracucci
gouthamve and pracucci authored Mar 9, 2020
1 parent c84d0fb commit a2ba001
Showing 1 changed file with 17 additions and 117 deletions.
134 changes: 17 additions & 117 deletions schema_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
yaml "gopkg.in/yaml.v2"

"github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/flagext"
)

const (
Expand All @@ -27,6 +26,7 @@ const (
var (
errInvalidSchemaVersion = errors.New("invalid schema version")
errInvalidTablePeriod = errors.New("the table period must be a multiple of 24h (1h for schema v1)")
errConfigFileNotSet = errors.New("schema config file needs to be set")
)

// PeriodConfig defines the schema and tables to use for a period of time
Expand Down Expand Up @@ -69,116 +69,31 @@ func (d *DayTime) UnmarshalYAML(unmarshal func(interface{}) error) error {
type SchemaConfig struct {
Configs []PeriodConfig `yaml:"configs"`

fileName string
legacy LegacySchemaConfig // if fileName is set then legacy config is ignored
}

// LegacySchemaConfig lets you configure schema via command-line flags
type LegacySchemaConfig struct {
StorageClient string // aws, gcp, etc.

// After midnight on this day, we start bucketing indexes by day instead of by
// hour. Only the day matters, not the time within the day.
DailyBucketsFrom flagext.DayValue
Base64ValuesFrom flagext.DayValue
V4SchemaFrom flagext.DayValue
V5SchemaFrom flagext.DayValue
V6SchemaFrom flagext.DayValue
V9SchemaFrom flagext.DayValue
BigtableColumnKeyFrom flagext.DayValue

// Config for the index & chunk tables.
OriginalTableName string
UsePeriodicTables bool
IndexTablesFrom flagext.DayValue
IndexTables PeriodicTableConfig
ChunkTablesFrom flagext.DayValue
ChunkTables PeriodicTableConfig
fileName string
legacyFileName string
}

// RegisterFlags adds the flags required to config this to the given FlagSet.
func (cfg *SchemaConfig) RegisterFlags(f *flag.FlagSet) {
flag.StringVar(&cfg.fileName, "config-yaml", "", "Schema config yaml")
cfg.legacy.RegisterFlags(f)
}

// RegisterFlags adds the flags required to config this to the given FlagSet.
func (cfg *LegacySchemaConfig) RegisterFlags(f *flag.FlagSet) {
flag.StringVar(&cfg.StorageClient, "chunk.storage-client", "aws", "Which storage client to use (aws, gcp, cassandra, inmemory).")
f.Var(&cfg.DailyBucketsFrom, "dynamodb.daily-buckets-from", "The date (in the format YYYY-MM-DD) of the first day for which DynamoDB index buckets should be day-sized vs. hour-sized.")
f.Var(&cfg.Base64ValuesFrom, "dynamodb.base64-buckets-from", "The date (in the format YYYY-MM-DD) after which we will stop querying to non-base64 encoded values.")
f.Var(&cfg.V4SchemaFrom, "dynamodb.v4-schema-from", "The date (in the format YYYY-MM-DD) after which we enable v4 schema.")
f.Var(&cfg.V5SchemaFrom, "dynamodb.v5-schema-from", "The date (in the format YYYY-MM-DD) after which we enable v5 schema.")
f.Var(&cfg.V6SchemaFrom, "dynamodb.v6-schema-from", "The date (in the format YYYY-MM-DD) after which we enable v6 schema.")
f.Var(&cfg.V9SchemaFrom, "dynamodb.v9-schema-from", "The date (in the format YYYY-MM-DD) after which we enable v9 schema (Series indexing).")
f.Var(&cfg.BigtableColumnKeyFrom, "bigtable.column-key-from", "The date (in the format YYYY-MM-DD) after which we use bigtable column keys.")

f.StringVar(&cfg.OriginalTableName, "dynamodb.original-table-name", "cortex", "The name of the DynamoDB table used before versioned schemas were introduced.")
f.BoolVar(&cfg.UsePeriodicTables, "dynamodb.use-periodic-tables", false, "Should we use periodic tables.")

f.Var(&cfg.IndexTablesFrom, "dynamodb.periodic-table.from", "Date after which to use periodic tables.")
cfg.IndexTables.RegisterFlags("dynamodb.periodic-table", "cortex_", f)
f.Var(&cfg.ChunkTablesFrom, "dynamodb.chunk-table.from", "Date after which to write chunks to DynamoDB.")
cfg.ChunkTables.RegisterFlags("dynamodb.chunk-table", "cortex_chunks_", f)
flag.StringVar(&cfg.fileName, "schema-config-file", "", "The path to the schema config file.")
// TODO(gouthamve): Add a metric for this.
flag.StringVar(&cfg.legacyFileName, "config-yaml", "", "DEPRECATED(use -schema-config-file) The path to the schema config file.")
}

func (cfg *SchemaConfig) loadFromFlags() error {
cfg.Configs = []PeriodConfig{}

add := func(t string, f model.Time) {
cfg.Configs = append(cfg.Configs, PeriodConfig{
From: DayTime{f},
Schema: t,
IndexType: cfg.legacy.StorageClient,
IndexTables: PeriodicTableConfig{
Prefix: cfg.legacy.OriginalTableName,
Tags: cfg.legacy.IndexTables.Tags,
},
})
}

add("v1", 0)
// loadFromFile loads the schema config from a yaml file
func (cfg *SchemaConfig) loadFromFile() error {
if cfg.fileName == "" {
cfg.fileName = cfg.legacyFileName

if cfg.legacy.DailyBucketsFrom.IsSet() {
add("v2", cfg.legacy.DailyBucketsFrom.Time)
}
if cfg.legacy.Base64ValuesFrom.IsSet() {
add("v3", cfg.legacy.Base64ValuesFrom.Time)
}
if cfg.legacy.V4SchemaFrom.IsSet() {
add("v4", cfg.legacy.V4SchemaFrom.Time)
}
if cfg.legacy.V5SchemaFrom.IsSet() {
add("v5", cfg.legacy.V5SchemaFrom.Time)
}
if cfg.legacy.V6SchemaFrom.IsSet() {
add("v6", cfg.legacy.V6SchemaFrom.Time)
}
if cfg.legacy.V9SchemaFrom.IsSet() {
add("v9", cfg.legacy.V9SchemaFrom.Time)
if cfg.legacyFileName != "" {
level.Warn(util.Logger).Log("msg", "running with DEPRECATED flag -config-yaml, use -schema-config-file instead")
}
}

cfg.ForEachAfter(cfg.legacy.IndexTablesFrom.Time, func(config *PeriodConfig) {
config.IndexTables = cfg.legacy.IndexTables
})
if cfg.legacy.ChunkTablesFrom.IsSet() {
cfg.ForEachAfter(cfg.legacy.ChunkTablesFrom.Time, func(config *PeriodConfig) {
if config.IndexType == "aws" {
config.IndexType = "aws-dynamo"
}
config.ChunkTables = cfg.legacy.ChunkTables
})
}
if cfg.legacy.BigtableColumnKeyFrom.IsSet() {
cfg.ForEachAfter(cfg.legacy.BigtableColumnKeyFrom.Time, func(config *PeriodConfig) {
config.IndexType = "gcp-columnkey"
})
if cfg.fileName == "" {
return errConfigFileNotSet
}
return nil
}

// loadFromFile loads the schema config from a yaml file
func (cfg *SchemaConfig) loadFromFile() error {
f, err := os.Open(cfg.fileName)
if err != nil {
return err
Expand Down Expand Up @@ -304,16 +219,8 @@ func (cfg *SchemaConfig) Load() error {
return nil
}

// Load config from file (if provided), falling back to CLI flags
var err error

if cfg.fileName == "" {
err = cfg.loadFromFlags()
} else {
err = cfg.loadFromFile()
}

if err != nil {
// Load config from file.
if err := cfg.loadFromFile(); err != nil {
return err
}

Expand Down Expand Up @@ -388,13 +295,6 @@ type PeriodicTableConfig struct {
Tags Tags `yaml:"tags,omitempty"`
}

// RegisterFlags adds the flags required to config this to the given FlagSet.
func (cfg *PeriodicTableConfig) RegisterFlags(argPrefix, tablePrefix string, f *flag.FlagSet) {
f.StringVar(&cfg.Prefix, argPrefix+".prefix", tablePrefix, "DynamoDB table prefix for period tables.")
f.DurationVar(&cfg.Period, argPrefix+".period", 7*24*time.Hour, "DynamoDB table period.")
f.Var(&cfg.Tags, argPrefix+".tag", "Tag (of the form key=value) to be added to all tables under management.")
}

// AutoScalingConfig for DynamoDB tables.
type AutoScalingConfig struct {
Enabled bool `yaml:"enabled,omitempty"`
Expand Down

0 comments on commit a2ba001

Please sign in to comment.