make fetch optimizations optional

Author: Dieterbe

api/config.go

@@ -28,6 +28,7 @@ var (
 	getTargetsConcurrency int
 	tagdbDefaultLimit     uint
 	speculationThreshold  float64
+	fetchOptimizations    bool
 
 	graphiteProxy *httputil.ReverseProxy
 	timeZone      *time.Location
@@ -49,6 +50,7 @@ func ConfigSetup() {
 	apiCfg.IntVar(&getTargetsConcurrency, "get-targets-concurrency", 20, "maximum number of concurrent threads for fetching data on the local node. Each thread handles a single series.")
 	apiCfg.UintVar(&tagdbDefaultLimit, "tagdb-default-limit", 100, "default limit for tagdb query results, can be overridden with query parameter \"limit\"")
 	apiCfg.Float64Var(&speculationThreshold, "speculation-threshold", 1, "ratio of peer responses after which speculation is used. Set to 1 to disable.")
+	apiCfg.BoolVar(&fetchOptimizations, "fetch-optimization", false, "enable MaxDataPoints optimization and pre-normalization optimizations")
 	globalconf.Register("http", apiCfg, flag.ExitOnError)
 }
 

api/graphite.go

@@ -225,7 +225,7 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 		// as graphite needs high-res data to perform its processing.
 		mdp = 0
 	}
-	plan, err := expr.NewPlan(exprs, fromUnix, toUnix, mdp, stable)
+	plan, err := expr.NewPlan(exprs, fromUnix, toUnix, mdp, stable, fetchOptimizations)
 	if err != nil {
 		if fun, ok := err.(expr.ErrUnknownFunction); ok {
 			if request.NoProxy {
@@ -1388,7 +1388,7 @@ func (s *Server) showPlan(ctx *middleware.Context, request models.GraphiteRender
 	stable := request.Process == "stable"
 	mdp := request.MaxDataPoints
 
-	plan, err := expr.NewPlan(exprs, fromUnix, toUnix, mdp, stable)
+	plan, err := expr.NewPlan(exprs, fromUnix, toUnix, mdp, stable, fetchOptimizations)
 	if err != nil {
 		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
 		return

expr/funcs.go

@@ -7,11 +7,12 @@ import (
 
 // Context describes a series timeframe and consolidator
 type Context struct {
-	from    uint32
-	to      uint32
-	consol  consolidation.Consolidator // can be 0 to mean undefined
-	PNGroup models.PNGroup             // pre-normalization group. if the data can be safely pre-normalized
-	MDP     uint32                     // if we can MDP-optimize, reflects runtime consolidation MaxDataPoints. 0 otherwise
+	from               uint32
+	to                 uint32
+	consol             consolidation.Consolidator // can be 0 to mean undefined
+	PNGroup            models.PNGroup             // pre-normalization group. if the data can be safely pre-normalized
+	MDP                uint32                     // if we can MDP-optimize, reflects runtime consolidation MaxDataPoints. 0 otherwise
+	fetchOptimizations bool                       // disable PNGroups and MDP
 }
 
 // GraphiteFunc defines a graphite processing function

expr/plan.go

@@ -34,14 +34,17 @@ func NewReq(query string, from, to uint32, cons consolidation.Consolidator) Req
 }
 
 func NewReqFromContext(query string, c Context) Req {
-	return Req{
-		Query:   query,
-		From:    c.from,
-		To:      c.to,
-		Cons:    c.consol,
-		PNGroup: c.PNGroup,
-		MDP:     c.MDP,
+	r := Req{
+		Query: query,
+		From:  c.from,
+		To:    c.to,
+		Cons:  c.consol,
+	}
+	if c.fetchOptimizations {
+		r.PNGroup = c.PNGroup
+		r.MDP = c.MDP
 	}
+	return r
 }
 
 // NewReqFromSeries generates a Req back from a series
@@ -105,7 +108,7 @@ func (p Plan) Dump(w io.Writer) {
 // * validation of arguments
 // * allow functions to modify the Context (change data range or consolidation)
 // * future version: allow functions to mark safe to pre-aggregate using consolidateBy or not
-func NewPlan(exprs []*expr, from, to, mdp uint32, stable bool) (Plan, error) {
+func NewPlan(exprs []*expr, from, to, mdp uint32, stable, fetchOptimizations bool) (Plan, error) {
 	plan := Plan{
 		exprs:         exprs,
 		MaxDataPoints: mdp,
@@ -114,10 +117,11 @@ func NewPlan(exprs []*expr, from, to, mdp uint32, stable bool) (Plan, error) {
 	}
 	for _, e := range exprs {
 		context := Context{
-			from:    from,
-			to:      to,
-			MDP:     mdp,
-			PNGroup: 0, // making this explicit here for easy code grepping
+			from:               from,
+			to:                 to,
+			MDP:                mdp,
+			PNGroup:            0, // making this explicit here for easy code grepping
+			fetchOptimizations: fetchOptimizations,
 		}
 		fn, reqs, err := newplan(e, context, stable, plan.Reqs)
 		if err != nil {

metrictank-sample.ini

@@ -248,6 +248,8 @@ get-targets-concurrency = 20
 tagdb-default-limit = 100
 # ratio of peer responses after which speculative querying (aka spec-exec) is used. Set to 1 to disable.
 speculation-threshold = 1
+# enable MaxDataPoints optimization and pre-normalization optimizations
+fetch-optimizations = false
 
 ## metric data inputs ##