Merge pull request #1127 from grafana/yaml

Underpinnings of a better single-process, no-dependancies Cortex.

cmd/alertmanager/main.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/alertmanager"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/go-kit/kit/log/level"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
@@ -36,7 +37,7 @@ func main() {
 		}
 		alertmanagerConfig alertmanager.MultitenantAlertmanagerConfig
 	)
-	util.RegisterFlags(&serverConfig, &alertmanagerConfig)
+	flagext.RegisterFlags(&serverConfig, &alertmanagerConfig)
 	flag.Parse()
 
 	util.InitLogger(&serverConfig)

cmd/configs/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/configs/api"
 	"github.com/cortexproject/cortex/pkg/configs/db"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
 )
@@ -26,7 +27,7 @@ func main() {
 		}
 		dbConfig db.Config
 	)
-	util.RegisterFlags(&serverConfig, &dbConfig)
+	flagext.RegisterFlags(&serverConfig, &dbConfig)
 	flag.Parse()
 
 	util.InitLogger(&serverConfig)

cmd/distributor/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/ingester/client"
 	"github.com/cortexproject/cortex/pkg/ring"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
@@ -53,7 +54,7 @@ func main() {
 		limits            validation.Limits
 		preallocConfig    client.PreallocConfig
 	)
-	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &clientConfig, &limits,
+	flagext.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &clientConfig, &limits,
 		&preallocConfig)
 	flag.Parse()
 

cmd/ingester/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/ingester"
 	"github.com/cortexproject/cortex/pkg/ingester/client"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
@@ -61,7 +62,7 @@ func main() {
 	// Ingester needs to know our gRPC listen port.
 	ingesterConfig.LifecyclerConfig.ListenPort = &serverConfig.GRPCListenPort
 
-	util.RegisterFlags(&serverConfig, &chunkStoreConfig, &storageConfig,
+	flagext.RegisterFlags(&serverConfig, &chunkStoreConfig, &storageConfig,
 		&schemaConfig, &ingesterConfig, &clientConfig, &limits, &preallocConfig, &marshalConfig)
 	flag.UintVar(&maxStreams, "ingester.max-concurrent-streams", 1000, "Limit on the number of concurrent streams for gRPC calls (0 = unlimited)")
 	flag.IntVar(&eventSampleRate, "event.sample-rate", 0, "How often to sample observability events (0 = never).")

cmd/lite/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/ring"
 	"github.com/cortexproject/cortex/pkg/ruler"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
@@ -85,7 +86,6 @@ func main() {
 	}
 	prometheus.MustRegister(r)
 	defer r.Stop()
-	ingesterConfig.LifecyclerConfig.KVClient = r.KVClient
 
 	dist, err := distributor.New(distributorConfig, ingesterClientConfig, overrides, r)
 	if err != nil {
@@ -209,7 +209,7 @@ func getConfigsFromCommandLine() {
 	}
 	// Ingester needs to know our gRPC listen port.
 	ingesterConfig.LifecyclerConfig.ListenPort = &serverConfig.GRPCListenPort
-	util.RegisterFlags(&serverConfig, &chunkStoreConfig, &distributorConfig, &querierConfig,
+	flagext.RegisterFlags(&serverConfig, &chunkStoreConfig, &distributorConfig, &querierConfig,
 		&ingesterConfig, &configStoreConfig, &rulerConfig, &storageConfig, &schemaConfig,
 		&ingesterClientConfig, &limitsConfig, &tbmConfig)
 	flag.BoolVar(&unauthenticated, "unauthenticated", false, "Set to true to disable multitenancy.")

cmd/querier/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/querier/frontend"
 	"github.com/cortexproject/cortex/pkg/ring"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	httpgrpc_server "github.com/weaveworks/common/httpgrpc/server"
 	"github.com/weaveworks/common/middleware"
@@ -49,7 +50,7 @@ func main() {
 		workerConfig      frontend.WorkerConfig
 		queryParallelism  int
 	)
-	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &clientConfig, &limits,
+	flagext.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &clientConfig, &limits,
 		&querierConfig, &chunkStoreConfig, &schemaConfig, &storageConfig, &workerConfig)
 	flag.IntVar(&queryParallelism, "querier.query-parallelism", 100, "Max subqueries run in parallel per higher-level query.")
 	flag.Parse()

cmd/query-frontend/main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/querier/frontend"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
 	"github.com/weaveworks/common/tracing"
@@ -25,7 +26,7 @@ func main() {
 		frontendConfig frontend.Config
 		maxMessageSize int
 	)
-	util.RegisterFlags(&serverConfig, &frontendConfig)
+	flagext.RegisterFlags(&serverConfig, &frontendConfig)
 	flag.IntVar(&maxMessageSize, "query-frontend.max-recv-message-size-bytes", 1024*1024*64, "Limit on the size of a grpc message this server can receive.")
 	flag.Parse()
 

cmd/ruler/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/ring"
 	"github.com/cortexproject/cortex/pkg/ruler"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
@@ -47,7 +48,7 @@ func main() {
 	trace := tracing.NewFromEnv("ruler")
 	defer trace.Close()
 
-	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &clientConfig, &limits,
+	flagext.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &clientConfig, &limits,
 		&rulerConfig, &chunkStoreConfig, &storageConfig, &schemaConfig, &configStoreConfig,
 		&querierConfig)
 	flag.Parse()

cmd/table-manager/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/chunk/storage"
 	"github.com/cortexproject/cortex/pkg/ingester"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/weaveworks/common/middleware"
 	"github.com/weaveworks/common/server"
 )
@@ -29,7 +30,7 @@ func main() {
 		schemaConfig   chunk.SchemaConfig
 		tbmConfig      chunk.TableManagerConfig
 	)
-	util.RegisterFlags(&ingesterConfig, &serverConfig, &storageConfig, &schemaConfig, &tbmConfig)
+	flagext.RegisterFlags(&ingesterConfig, &serverConfig, &storageConfig, &schemaConfig, &tbmConfig)
 	flag.Parse()
 
 	util.InitLogger(&serverConfig)

cmd/test-exporter/main.go

@@ -8,11 +8,12 @@ import (
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/weaveworks/common/server"
+	"github.com/weaveworks/common/tracing"
 
 	"github.com/cortexproject/cortex/pkg/querier/correctness"
 	"github.com/cortexproject/cortex/pkg/util"
-	"github.com/weaveworks/common/server"
-	"github.com/weaveworks/common/tracing"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
 var (
@@ -24,8 +25,7 @@ func main() {
 		serverConfig server.Config
 		runnerConfig correctness.RunnerConfig
 	)
-	serverConfig.RegisterFlags(flag.CommandLine)
-	runnerConfig.RegisterFlags(flag.CommandLine)
+	flagext.RegisterFlags(&serverConfig, &runnerConfig)
 	flag.Parse()
 
 	// Setting the environment variable JAEGER_AGENT_HOST enables tracing

pkg/alertmanager/multitenant.go

@@ -22,6 +22,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/configs"
 	configs_client "github.com/cortexproject/cortex/pkg/configs/client"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/weaveworks/common/instrument"
 	"github.com/weaveworks/common/user"
 	"github.com/weaveworks/mesh"
@@ -176,8 +177,8 @@ func counts(counts map[string]int, keys []string) string {
 type MultitenantAlertmanagerConfig struct {
 	DataDir       string
 	Retention     time.Duration
-	ExternalURL   util.URLValue
-	ConfigsAPIURL util.URLValue
+	ExternalURL   flagext.URLValue
+	ConfigsAPIURL flagext.URLValue
 	PollInterval  time.Duration
 	ClientTimeout time.Duration
 

pkg/chunk/aws/storage_client.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/chunk"
 	chunk_util "github.com/cortexproject/cortex/pkg/chunk/util"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	awscommon "github.com/weaveworks/common/aws"
 	"github.com/weaveworks/common/instrument"
 	"github.com/weaveworks/common/user"
@@ -97,9 +98,9 @@ func init() {
 
 // DynamoDBConfig specifies config for a DynamoDB database.
 type DynamoDBConfig struct {
-	DynamoDB               util.URLValue
+	DynamoDB               flagext.URLValue
 	APILimit               float64
-	ApplicationAutoScaling util.URLValue
+	ApplicationAutoScaling flagext.URLValue
 	Metrics                MetricsAutoScalingConfig
 	ChunkGangSize          int
 	ChunkGetMaxParallelism int
@@ -123,7 +124,7 @@ func (cfg *DynamoDBConfig) RegisterFlags(f *flag.FlagSet) {
 // StorageConfig specifies config for storing data on AWS.
 type StorageConfig struct {
 	DynamoDBConfig
-	S3 util.URLValue
+	S3 flagext.URLValue
 }
 
 // RegisterFlags adds the flags required to config this to the given FlagSet

pkg/chunk/chunk_store_test.go

@@ -17,8 +17,8 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/chunk/cache"
 	"github.com/cortexproject/cortex/pkg/chunk/encoding"
-	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/extract"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/weaveworks/common/test"
 	"github.com/weaveworks/common/user"
@@ -49,7 +49,7 @@ var stores = []struct {
 			var (
 				storeCfg StoreConfig
 			)
-			util.DefaultValues(&storeCfg)
+			flagext.DefaultValues(&storeCfg)
 			return storeCfg
 		},
 	},
@@ -59,7 +59,7 @@ var stores = []struct {
 			var (
 				storeCfg StoreConfig
 			)
-			util.DefaultValues(&storeCfg)
+			flagext.DefaultValues(&storeCfg)
 
 			storeCfg.WriteDedupeCacheConfig.Cache = cache.NewFifoCache("test", cache.FifoCacheConfig{
 				Size: 500,
@@ -75,7 +75,7 @@ func newTestChunkStore(t *testing.T, schemaName string) Store {
 	var (
 		storeCfg StoreConfig
 	)
-	util.DefaultValues(&storeCfg)
+	flagext.DefaultValues(&storeCfg)
 	return newTestChunkStoreConfig(t, schemaName, storeCfg)
 }
 
@@ -84,7 +84,7 @@ func newTestChunkStoreConfig(t *testing.T, schemaName string, storeCfg StoreConf
 		tbmConfig TableManagerConfig
 		schemaCfg = DefaultSchemaConfig("", schemaName, 0)
 	)
-	util.DefaultValues(&tbmConfig)
+	flagext.DefaultValues(&tbmConfig)
 	storage := NewMockStorage()
 	tableManager, err := NewTableManager(tbmConfig, schemaCfg, maxChunkAge, storage)
 	require.NoError(t, err)
@@ -93,7 +93,7 @@ func newTestChunkStoreConfig(t *testing.T, schemaName string, storeCfg StoreConf
 	require.NoError(t, err)
 
 	var limits validation.Limits
-	util.DefaultValues(&limits)
+	flagext.DefaultValues(&limits)
 	limits.MaxQueryLength = 30 * 24 * time.Hour
 	overrides, err := validation.NewOverrides(limits)
 	require.NoError(t, err)

pkg/chunk/schema_config.go

@@ -11,6 +11,7 @@ import (
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/weaveworks/common/mtime"
 )
 
@@ -45,20 +46,20 @@ type LegacySchemaConfig struct {
 
 	// After midnight on this day, we start bucketing indexes by day instead of by
 	// hour.  Only the day matters, not the time within the day.
-	DailyBucketsFrom      util.DayValue
-	Base64ValuesFrom      util.DayValue
-	V4SchemaFrom          util.DayValue
-	V5SchemaFrom          util.DayValue
-	V6SchemaFrom          util.DayValue
-	V9SchemaFrom          util.DayValue
-	BigtableColumnKeyFrom util.DayValue
+	DailyBucketsFrom      flagext.DayValue
+	Base64ValuesFrom      flagext.DayValue
+	V4SchemaFrom          flagext.DayValue
+	V5SchemaFrom          flagext.DayValue
+	V6SchemaFrom          flagext.DayValue
+	V9SchemaFrom          flagext.DayValue
+	BigtableColumnKeyFrom flagext.DayValue
 
 	// Config for the index & chunk tables.
 	OriginalTableName string
 	UsePeriodicTables bool
-	IndexTablesFrom   util.DayValue
+	IndexTablesFrom   flagext.DayValue
 	IndexTables       PeriodicTableConfig
-	ChunkTablesFrom   util.DayValue
+	ChunkTablesFrom   flagext.DayValue
 	ChunkTables       PeriodicTableConfig
 }
 

pkg/chunk/storage/factory_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cortexproject/cortex/pkg/chunk"
-	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
@@ -18,7 +18,7 @@ func TestFactoryStop(t *testing.T) {
 		schemaConfig chunk.SchemaConfig
 		defaults     validation.Limits
 	)
-	util.DefaultValues(&cfg, &storeConfig, &schemaConfig, &defaults)
+	flagext.DefaultValues(&cfg, &storeConfig, &schemaConfig, &defaults)
 	schemaConfig.Configs = []chunk.PeriodConfig{
 		{
 			From:  model.Time(0),

pkg/chunk/testutils/testutils.go

@@ -6,10 +6,10 @@ import (
 	"time"
 
 	promchunk "github.com/cortexproject/cortex/pkg/chunk/encoding"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/prometheus/common/model"
 
 	"github.com/cortexproject/cortex/pkg/chunk"
-	"github.com/cortexproject/cortex/pkg/util"
 )
 
 const (
@@ -26,7 +26,7 @@ type Fixture interface {
 // Setup a fixture with initial tables
 func Setup(fixture Fixture, tableName string) (chunk.StorageClient, error) {
 	var tbmConfig chunk.TableManagerConfig
-	util.DefaultValues(&tbmConfig)
+	flagext.DefaultValues(&tbmConfig)
 	storageClient, tableClient, schemaConfig, err := fixture.Clients()
 	if err != nil {
 		return nil, err

pkg/distributor/distributor_test.go

@@ -23,8 +23,8 @@ import (
 	"github.com/cortexproject/cortex/pkg/chunk/encoding"
 	"github.com/cortexproject/cortex/pkg/ingester/client"
 	"github.com/cortexproject/cortex/pkg/ring"
-	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/chunkcompat"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/user"
@@ -278,7 +278,7 @@ func prepare(t *testing.T, numIngesters, happyIngesters int, queryDelay time.Dur
 	var cfg Config
 	var limits validation.Limits
 	var clientConfig client.Config
-	util.DefaultValues(&cfg, &limits, &clientConfig)
+	flagext.DefaultValues(&cfg, &limits, &clientConfig)
 	limits.IngestionRate = 20
 	limits.IngestionBurstSize = 20
 	cfg.ingesterClientFactory = factory