Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
api: enforce tenant on generated time series
Browse files Browse the repository at this point in the history
Currently, recording and alerting rules that are fetched from the new
rules/raw API will not have any tenant label attached to them. This
means that when the rules are configured on a Thanos Ruler, the time
series that are generated will not have a tenant label on them. This is
a big problem, because without this tenant label, the very tenant to
whom the rules belong cannot query for the time series, making them in
effect useless. This commit enforces that all rules returned by the
rules/raw endpoint have a tenant_id label attached to them.

Note: this is a distinct problem from ensuring that the rules written by
a tenant are guaranteed to only include data from that tenant.
Currently, the rules from a tenant can select data from any tenant. This
is a distinct problem and should absolutely be tackled in a follow up.

Signed-off-by: Lucas Servén Marín <lserven@gmail.com>
squat committed Nov 14, 2021
1 parent 41492fe commit 16b16c7
Showing 3 changed files with 70 additions and 5 deletions.
10 changes: 9 additions & 1 deletion api/metrics/v1/http.go
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@ type handlerConfiguration struct {
registry *prometheus.Registry
instrument handlerInstrumenter
spanRoutePrefix string
tenantLabel string
queryMiddlewares []func(http.Handler) http.Handler
readMiddlewares []func(http.Handler) http.Handler
uiMiddlewares []func(http.Handler) http.Handler
@@ -77,6 +78,13 @@ func WithSpanRoutePrefix(spanRoutePrefix string) HandlerOption {
}
}

// WithTenantLabel adds tenant label for the handler to use.
func WithTenantLabel(tenantLabel string) HandlerOption {
return func(h *handlerConfiguration) {
h.tenantLabel = tenantLabel
}
}

// WithReadMiddleware adds a middleware for all "matcher based" read operations (series, label names and values).
func WithReadMiddleware(m func(http.Handler) http.Handler) HandlerOption {
return func(h *handlerConfiguration) {
@@ -284,7 +292,7 @@ func NewHandler(read, write, rulesEndpoint *url.URL, upstreamCA []byte, opts ...
return r
}

rh := rulesHandler{client: client}
rh := rulesHandler{client: client, logger: c.logger, tenantLabel: c.tenantLabel}

r.Group(func(r chi.Router) {
r.Use(c.uiMiddlewares...)
64 changes: 60 additions & 4 deletions api/metrics/v1/rules.go
Original file line number Diff line number Diff line change
@@ -2,14 +2,20 @@ package v1

import (
"io"
"io/ioutil"
"net/http"

"github.com/ghodss/yaml"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/observatorium/api/authentication"
"github.com/observatorium/api/rules"
)

type rulesHandler struct {
client rules.ClientInterface
client rules.ClientInterface
logger log.Logger
tenantLabel string
}

func (rh *rulesHandler) get(w http.ResponseWriter, r *http.Request) {
@@ -19,8 +25,15 @@ func (rh *rulesHandler) get(w http.ResponseWriter, r *http.Request) {
return
}

id, ok := authentication.GetTenantID(r.Context())
if !ok {
http.Error(w, "error finding tenant ID", http.StatusUnauthorized)
return
}

resp, err := rh.client.ListRules(r.Context(), tenant)
if err != nil {
level.Error(rh.logger).Log("msg", "could not list rules", "err", err.Error())
sc := http.StatusInternalServerError
if resp != nil {
sc = resp.StatusCode
@@ -31,10 +44,52 @@ func (rh *rulesHandler) get(w http.ResponseWriter, r *http.Request) {
return
}

defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
http.Error(w, "error listing rules", resp.StatusCode)
return
}

if _, err := io.Copy(w, resp.Body); err != nil {
http.Error(w, "error writing rules response", http.StatusInternalServerError)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
http.Error(w, "error listing rules", http.StatusInternalServerError)
return
}

var rawRules rules.Rules
if err := yaml.Unmarshal(body, &rawRules); err != nil {
level.Error(rh.logger).Log("msg", "could not unmarshal rules", "err", err.Error())
http.Error(w, "error unmarshaling rules", http.StatusInternalServerError)
return
}

for i := range rawRules.Groups {
for j := range rawRules.Groups[i].Rules {
switch r := rawRules.Groups[i].Rules[j].(type) {
case rules.RecordingRule:
if r.Labels.AdditionalProperties == nil {
r.Labels.AdditionalProperties = make(map[string]string)
}
r.Labels.AdditionalProperties[rh.tenantLabel] = id
rawRules.Groups[i].Rules[j] = r
case rules.AlertingRule:
if r.Labels.AdditionalProperties == nil {
r.Labels.AdditionalProperties = make(map[string]string)
}
r.Labels.AdditionalProperties[rh.tenantLabel] = id
rawRules.Groups[i].Rules[j] = r
}
}
}

body, err = yaml.Marshal(rawRules)
if err != nil {
level.Error(rh.logger).Log("msg", "could not marshal YAML", "err", err.Error())
http.Error(w, "error marshaling YAML", http.StatusInternalServerError)
return
}

if _, err := w.Write(body); err != nil {
level.Error(rh.logger).Log("msg", "could not write body", "err", err.Error())
return
}
}
@@ -52,6 +107,7 @@ func (rh *rulesHandler) put(w http.ResponseWriter, r *http.Request) {
sc = resp.StatusCode
}

level.Error(rh.logger).Log("msg", "could not set rules", "err", err.Error())
http.Error(w, "error creating rules", sc)

return
1 change: 1 addition & 0 deletions main.go
Original file line number Diff line number Diff line change
@@ -548,6 +548,7 @@ func main() {
metricsv1.WithRegistry(reg),
metricsv1.WithHandlerInstrumenter(ins),
metricsv1.WithSpanRoutePrefix("/api/metrics/v1/{tenant}"),
metricsv1.WithTenantLabel(cfg.metrics.tenantLabel),
metricsv1.WithQueryMiddleware(authorization.WithAuthorizers(authorizers, rbac.Read, "metrics")),
metricsv1.WithQueryMiddleware(metricsv1.WithEnforceTenancyOnQuery(cfg.metrics.tenantLabel)),
metricsv1.WithReadMiddleware(authorization.WithAuthorizers(authorizers, rbac.Read, "metrics")),

0 comments on commit 16b16c7

Please sign in to comment.