diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a437992eb..d31b9d8746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * [ENHANCEMENT] Query-frontend/scheduler: add a new limit `frontend.max-outstanding-requests-per-tenant` for configuring queue size per tenant. Started deprecating two flags `-query-scheduler.max-outstanding-requests-per-tenant` and `-querier.max-outstanding-requests-per-tenant`, and change their value default to 0. Now if both the old flag and new flag are specified, the old flag's queue size will be picked. #5005 * [ENHANCEMENT] Query-tee: Add `/api/v1/query_exemplars` API endpoint support. #5010 * [ENHANCEMENT] Query Frontend/Query Scheduler: Increase upper bound to 60s for queue duration histogram metric. #5029 +* [ENHANCEMENT] Query Frontend: Log Vertical sharding information when `query_stats_enabled` is enabled. #5037 * [FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. #4978 * [FEATURE] Ingester: Add active series to all_user_stats page. #4972 * [FEATURE] Ingester: Added `-blocks-storage.tsdb.head-chunks-write-queue-size` allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000 diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 99f2116e5b..794f61cf38 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -112,7 +112,7 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var ( - stats *querier_stats.Stats + stats *querier_stats.QueryStats queryString url.Values ) @@ -185,7 +185,7 @@ func (f *Handler) reportSlowQuery(r *http.Request, queryString url.Values, query level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } -func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, queryResponseTime time.Duration, stats *querier_stats.Stats, error error) { +func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, queryResponseTime time.Duration, stats *querier_stats.QueryStats, error error) { tenantIDs, err := tenant.TenantIDs(r.Context()) if err != nil { return @@ -214,7 +214,9 @@ func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, quer "fetched_series_count", numSeries, "fetched_chunks_bytes", numBytes, "fetched_data_bytes", numDataBytes, - }, formatQueryString(queryString)...) + }, stats.LoadExtraFields()...) + + logMessage = append(logMessage, formatQueryString(queryString)...) if error != nil { s, ok := status.FromError(error) @@ -264,7 +266,7 @@ func writeError(w http.ResponseWriter, err error) { server.WriteError(w, err) } -func writeServiceTimingHeader(queryResponseTime time.Duration, headers http.Header, stats *querier_stats.Stats) { +func writeServiceTimingHeader(queryResponseTime time.Duration, headers http.Header, stats *querier_stats.QueryStats) { if stats != nil { parts := make([]string, 0) parts = append(parts, statsValue("querier_wall_time", stats.LoadWallTime())) diff --git a/pkg/frontend/v1/frontendv1pb/frontend.pb.go b/pkg/frontend/v1/frontendv1pb/frontend.pb.go index b8480261ed..6e8a9be191 100644 --- a/pkg/frontend/v1/frontendv1pb/frontend.pb.go +++ b/pkg/frontend/v1/frontendv1pb/frontend.pb.go @@ -9,7 +9,8 @@ package frontendv1pb import ( context "context" fmt "fmt" - stats "github.com/cortexproject/cortex/pkg/querier/stats" + _ "github.com/cortexproject/cortex/pkg/querier/stats" + github_com_cortexproject_cortex_pkg_querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" httpgrpc "github.com/weaveworks/common/httpgrpc" @@ -118,9 +119,9 @@ func (m *FrontendToClient) GetStatsEnabled() bool { } type ClientToFrontend struct { - HttpResponse *httpgrpc.HTTPResponse `protobuf:"bytes,1,opt,name=httpResponse,proto3" json:"httpResponse,omitempty"` - ClientID string `protobuf:"bytes,2,opt,name=clientID,proto3" json:"clientID,omitempty"` - Stats *stats.Stats `protobuf:"bytes,3,opt,name=stats,proto3" json:"stats,omitempty"` + HttpResponse *httpgrpc.HTTPResponse `protobuf:"bytes,1,opt,name=httpResponse,proto3" json:"httpResponse,omitempty"` + ClientID string `protobuf:"bytes,2,opt,name=clientID,proto3" json:"clientID,omitempty"` + Stats *github_com_cortexproject_cortex_pkg_querier_stats.QueryStats `protobuf:"bytes,3,opt,name=stats,proto3,customtype=github.com/cortexproject/cortex/pkg/querier/stats.QueryStats" json:"stats,omitempty"` } func (m *ClientToFrontend) Reset() { *m = ClientToFrontend{} } @@ -169,13 +170,6 @@ func (m *ClientToFrontend) GetClientID() string { return "" } -func (m *ClientToFrontend) GetStats() *stats.Stats { - if m != nil { - return m.Stats - } - return nil -} - type NotifyClientShutdownRequest struct { ClientID string `protobuf:"bytes,1,opt,name=clientID,proto3" json:"clientID,omitempty"` } @@ -265,38 +259,40 @@ func init() { func init() { proto.RegisterFile("frontend.proto", fileDescriptor_eca3873955a29cfe) } var fileDescriptor_eca3873955a29cfe = []byte{ - // 496 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x6e, 0xd3, 0x40, - 0x10, 0x86, 0x3d, 0x50, 0x4a, 0x98, 0x44, 0x91, 0xb5, 0x02, 0x14, 0x19, 0xb4, 0x8a, 0x2c, 0x40, - 0x11, 0x12, 0x36, 0x04, 0x24, 0x04, 0x12, 0x97, 0xd2, 0x50, 0x7a, 0x41, 0xc5, 0x31, 0x17, 0x2e, - 0x55, 0xec, 0x6c, 0x9c, 0xd0, 0xc6, 0xeb, 0xda, 0xeb, 0x86, 0xdc, 0x78, 0x02, 0x84, 0xc4, 0x4b, - 0xf0, 0x0c, 0x3c, 0x01, 0xc7, 0x1c, 0x7b, 0x24, 0xce, 0x85, 0x63, 0x1f, 0x01, 0x65, 0xd7, 0x71, - 0x9d, 0xa8, 0x82, 0xcb, 0x6a, 0xc7, 0x33, 0xff, 0xcc, 0x37, 0xbf, 0x17, 0xeb, 0x83, 0x98, 0x87, - 0x82, 0x85, 0x7d, 0x2b, 0x8a, 0xb9, 0xe0, 0xa4, 0xb2, 0x8a, 0x8d, 0x47, 0xc1, 0x48, 0x0c, 0x53, - 0xcf, 0xf2, 0xf9, 0xd8, 0x0e, 0x78, 0xc0, 0x6d, 0x59, 0xe0, 0xa5, 0x03, 0x19, 0xc9, 0x40, 0xde, - 0x94, 0xd0, 0x78, 0x56, 0x2a, 0x9f, 0xb0, 0xde, 0x29, 0x9b, 0xf0, 0xf8, 0x28, 0xb1, 0x7d, 0x3e, - 0x1e, 0xf3, 0xd0, 0x1e, 0x0a, 0x11, 0x05, 0x71, 0xe4, 0x17, 0x97, 0x5c, 0xf5, 0xaa, 0xa4, 0xf2, - 0x79, 0x2c, 0xd8, 0xe7, 0x28, 0xe6, 0x9f, 0x98, 0x2f, 0xf2, 0xc8, 0x8e, 0x8e, 0x02, 0xfb, 0x24, - 0x65, 0xf1, 0x88, 0xc5, 0x76, 0x22, 0x7a, 0x22, 0x51, 0xa7, 0x92, 0x9b, 0xdf, 0x01, 0xf5, 0x37, - 0x39, 0xb0, 0xcb, 0x5f, 0x1f, 0x8f, 0x58, 0x28, 0xc8, 0x73, 0xac, 0x2e, 0xa7, 0x38, 0xec, 0x24, - 0x65, 0x89, 0x68, 0x40, 0x13, 0x5a, 0xd5, 0xf6, 0x2d, 0xab, 0x98, 0xfc, 0xd6, 0x75, 0x0f, 0xf2, - 0xa4, 0x53, 0xae, 0x24, 0x26, 0x6e, 0x89, 0x69, 0xc4, 0x1a, 0x57, 0x9a, 0xd0, 0xaa, 0xb7, 0xeb, - 0x56, 0x61, 0x8d, 0x3b, 0x8d, 0x98, 0x23, 0x73, 0xc4, 0xc4, 0x9a, 0x04, 0xe8, 0x84, 0x3d, 0xef, - 0x98, 0xf5, 0x1b, 0x57, 0x9b, 0xd0, 0xaa, 0x38, 0x6b, 0xdf, 0xcc, 0xaf, 0x80, 0xba, 0x62, 0x71, - 0xf9, 0x8a, 0x8e, 0xbc, 0xc4, 0x9a, 0x9a, 0x95, 0x44, 0x3c, 0x4c, 0x58, 0x8e, 0x75, 0x7b, 0x13, - 0x4b, 0x65, 0x9d, 0xb5, 0x5a, 0x62, 0x60, 0xc5, 0x97, 0xfd, 0xf6, 0x77, 0x25, 0xdc, 0x0d, 0xa7, - 0x88, 0x89, 0x89, 0xd7, 0xe4, 0x70, 0x49, 0x52, 0x6d, 0xd7, 0x2c, 0xe5, 0x4f, 0x77, 0x79, 0x3a, - 0x2a, 0x65, 0xbe, 0xc0, 0x3b, 0xef, 0xb8, 0x18, 0x0d, 0xa6, 0x8a, 0xaa, 0x3b, 0x4c, 0x45, 0x9f, - 0x4f, 0xc2, 0xd5, 0xde, 0xe5, 0xf6, 0xb0, 0xde, 0xde, 0xa4, 0x78, 0xf7, 0x72, 0xa9, 0x42, 0x7b, - 0x78, 0x0f, 0xb7, 0x96, 0xee, 0x10, 0x1d, 0x6b, 0xcb, 0x05, 0x0e, 0x9d, 0xce, 0xfb, 0x0f, 0x9d, - 0xae, 0xab, 0x6b, 0x04, 0x71, 0x7b, 0xaf, 0xe3, 0x1e, 0xee, 0xef, 0xea, 0xd0, 0xfe, 0x09, 0x58, - 0x29, 0x9c, 0xd8, 0xc3, 0xeb, 0x07, 0x31, 0xf7, 0x59, 0x92, 0x10, 0xe3, 0xc2, 0xe3, 0x4d, 0xc3, - 0x8c, 0x52, 0x6e, 0xf3, 0x17, 0x9b, 0x5a, 0x0b, 0x1e, 0x03, 0x61, 0x78, 0xf3, 0x32, 0x36, 0x72, - 0xff, 0x42, 0xf9, 0x8f, 0xb5, 0x8d, 0x07, 0xff, 0x2b, 0x53, 0x2b, 0xee, 0xec, 0xcc, 0xe6, 0x54, - 0x3b, 0x9b, 0x53, 0xed, 0x7c, 0x4e, 0xe1, 0x4b, 0x46, 0xe1, 0x47, 0x46, 0xe1, 0x57, 0x46, 0x61, - 0x96, 0x51, 0xf8, 0x9d, 0x51, 0xf8, 0x93, 0x51, 0xed, 0x3c, 0xa3, 0xf0, 0x6d, 0x41, 0xb5, 0xd9, - 0x82, 0x6a, 0x67, 0x0b, 0xaa, 0x7d, 0xac, 0xad, 0x9a, 0x9f, 0x3e, 0x89, 0x3c, 0x6f, 0x5b, 0xbe, - 0xd7, 0xa7, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x1f, 0xe0, 0x35, 0xe6, 0x6f, 0x03, 0x00, 0x00, + // 515 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0x41, 0x8b, 0xd3, 0x40, + 0x14, 0xc7, 0x33, 0xba, 0xae, 0xf5, 0xb5, 0x94, 0x30, 0xa8, 0x94, 0x28, 0x63, 0x09, 0x2a, 0x45, + 0x30, 0xd1, 0x2a, 0x88, 0xa2, 0x20, 0x75, 0xeb, 0xba, 0x17, 0xd9, 0x4d, 0xe3, 0xc5, 0xcb, 0xd2, + 0xa4, 0xd3, 0xb4, 0xee, 0x36, 0x93, 0x9d, 0x4c, 0xb6, 0xf6, 0xe6, 0x47, 0x10, 0xfc, 0x12, 0x7e, + 0x06, 0x3f, 0x81, 0xc7, 0x1e, 0x17, 0x0f, 0x62, 0xd3, 0x8b, 0xc7, 0xfd, 0x08, 0xd2, 0x99, 0x34, + 0x9b, 0x96, 0x45, 0xf1, 0x32, 0xcc, 0xcb, 0x7b, 0xff, 0xf7, 0x7e, 0xef, 0x9f, 0x81, 0x6a, 0x9f, + 0xb3, 0x50, 0xd0, 0xb0, 0x67, 0x45, 0x9c, 0x09, 0x86, 0x4b, 0xcb, 0xd8, 0xb8, 0x1f, 0x0c, 0xc5, + 0x20, 0xf1, 0x2c, 0x9f, 0x8d, 0xec, 0x80, 0x05, 0xcc, 0x96, 0x05, 0x5e, 0xd2, 0x97, 0x91, 0x0c, + 0xe4, 0x4d, 0x09, 0x8d, 0xc7, 0x85, 0xf2, 0x31, 0xed, 0x1e, 0xd3, 0x31, 0xe3, 0x07, 0xb1, 0xed, + 0xb3, 0xd1, 0x88, 0x85, 0xf6, 0x40, 0x88, 0x28, 0xe0, 0x91, 0x9f, 0x5f, 0x32, 0xd5, 0x8b, 0x82, + 0xca, 0x67, 0x5c, 0xd0, 0x8f, 0x11, 0x67, 0x1f, 0xa8, 0x2f, 0xb2, 0xc8, 0x8e, 0x0e, 0x02, 0xfb, + 0x28, 0xa1, 0x7c, 0x48, 0xb9, 0x1d, 0x8b, 0xae, 0x88, 0xd5, 0xa9, 0xe4, 0xe6, 0x17, 0x04, 0xfa, + 0xeb, 0x0c, 0xd8, 0x65, 0xaf, 0x0e, 0x87, 0x34, 0x14, 0xf8, 0x09, 0x94, 0x17, 0x53, 0x1c, 0x7a, + 0x94, 0xd0, 0x58, 0xd4, 0x50, 0x1d, 0x35, 0xca, 0xcd, 0x6b, 0x56, 0x3e, 0xf9, 0x8d, 0xeb, 0xee, + 0x66, 0x49, 0xa7, 0x58, 0x89, 0x4d, 0xd8, 0x10, 0x93, 0x88, 0xd6, 0x2e, 0xd4, 0x51, 0xa3, 0xda, + 0xac, 0x5a, 0xb9, 0x35, 0xee, 0x24, 0xa2, 0x8e, 0xcc, 0x61, 0x13, 0x2a, 0x12, 0xa0, 0x1d, 0x76, + 0xbd, 0x43, 0xda, 0xab, 0x5d, 0xac, 0xa3, 0x46, 0xc9, 0x59, 0xf9, 0x66, 0x4e, 0x11, 0xe8, 0x8a, + 0xc5, 0x65, 0x4b, 0x3a, 0xfc, 0x0c, 0x2a, 0x6a, 0x56, 0x1c, 0xb1, 0x30, 0xa6, 0x19, 0xd6, 0xf5, + 0x75, 0x2c, 0x95, 0x75, 0x56, 0x6a, 0xb1, 0x01, 0x25, 0x5f, 0xf6, 0xdb, 0xd9, 0x92, 0x70, 0x57, + 0x9c, 0x3c, 0xc6, 0x3d, 0xb8, 0x24, 0x87, 0x4b, 0x92, 0x72, 0xb3, 0x62, 0x29, 0x7f, 0x3a, 0x8b, + 0xb3, 0xf5, 0xf2, 0xc7, 0xcf, 0x5b, 0xcf, 0xff, 0xdb, 0x62, 0x6b, 0x2f, 0xa1, 0x7c, 0x22, 0x3b, + 0x38, 0xaa, 0xb9, 0xf9, 0x14, 0x6e, 0xbc, 0x65, 0x62, 0xd8, 0x9f, 0xa8, 0xbd, 0x3a, 0x83, 0x44, + 0xf4, 0xd8, 0x38, 0x5c, 0x3a, 0x57, 0x04, 0x44, 0xab, 0x80, 0x26, 0x81, 0x9b, 0xe7, 0x4b, 0xd5, + 0x72, 0xf7, 0x6e, 0xc3, 0xc6, 0xc2, 0x5f, 0xac, 0x43, 0x65, 0x61, 0xc1, 0xbe, 0xd3, 0xde, 0x7b, + 0xd7, 0xee, 0xb8, 0xba, 0x86, 0x01, 0x36, 0xb7, 0xdb, 0xee, 0xfe, 0xce, 0x96, 0x8e, 0x9a, 0xdf, + 0x10, 0x94, 0x72, 0x2f, 0xb7, 0xe1, 0xf2, 0x2e, 0x67, 0x3e, 0x8d, 0x63, 0x6c, 0x9c, 0xfd, 0xa5, + 0x75, 0xcb, 0x8d, 0x42, 0x6e, 0xfd, 0x91, 0x98, 0x5a, 0x03, 0x3d, 0x40, 0x98, 0xc2, 0xd5, 0xf3, + 0xd8, 0xf0, 0x9d, 0x33, 0xe5, 0x5f, 0xd6, 0x36, 0xee, 0xfe, 0xab, 0x4c, 0xad, 0xd8, 0x6a, 0x4d, + 0x67, 0x44, 0x3b, 0x99, 0x11, 0xed, 0x74, 0x46, 0xd0, 0xa7, 0x94, 0xa0, 0xaf, 0x29, 0x41, 0xdf, + 0x53, 0x82, 0xa6, 0x29, 0x41, 0xbf, 0x52, 0x82, 0x7e, 0xa7, 0x44, 0x3b, 0x4d, 0x09, 0xfa, 0x3c, + 0x27, 0xda, 0x74, 0x4e, 0xb4, 0x93, 0x39, 0xd1, 0xde, 0x57, 0x96, 0xcd, 0x8f, 0x1f, 0x46, 0x9e, + 0xb7, 0x29, 0x5f, 0xfc, 0xa3, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xac, 0x79, 0xcf, 0x36, 0xb1, + 0x03, 0x00, 0x00, } func (x Type) String() string { @@ -361,7 +357,11 @@ func (this *ClientToFrontend) Equal(that interface{}) bool { if this.ClientID != that1.ClientID { return false } - if !this.Stats.Equal(that1.Stats) { + if that1.Stats == nil { + if this.Stats != nil { + return false + } + } else if !this.Stats.Equal(*that1.Stats) { return false } return true @@ -435,9 +435,7 @@ func (this *ClientToFrontend) GoString() string { s = append(s, "HttpResponse: "+fmt.Sprintf("%#v", this.HttpResponse)+",\n") } s = append(s, "ClientID: "+fmt.Sprintf("%#v", this.ClientID)+",\n") - if this.Stats != nil { - s = append(s, "Stats: "+fmt.Sprintf("%#v", this.Stats)+",\n") - } + s = append(s, "Stats: "+fmt.Sprintf("%#v", this.Stats)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -696,11 +694,11 @@ func (m *ClientToFrontend) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = l if m.Stats != nil { { - size, err := m.Stats.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { + size := m.Stats.Size() + i -= size + if _, err := m.Stats.MarshalTo(dAtA[i:]); err != nil { return 0, err } - i -= size i = encodeVarintFrontend(dAtA, i, uint64(size)) } i-- @@ -879,7 +877,7 @@ func (this *ClientToFrontend) String() string { s := strings.Join([]string{`&ClientToFrontend{`, `HttpResponse:` + strings.Replace(fmt.Sprintf("%v", this.HttpResponse), "HTTPResponse", "httpgrpc.HTTPResponse", 1) + `,`, `ClientID:` + fmt.Sprintf("%v", this.ClientID) + `,`, - `Stats:` + strings.Replace(fmt.Sprintf("%v", this.Stats), "Stats", "stats.Stats", 1) + `,`, + `Stats:` + fmt.Sprintf("%v", this.Stats) + `,`, `}`, }, "") return s @@ -1166,7 +1164,7 @@ func (m *ClientToFrontend) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Stats == nil { - m.Stats = &stats.Stats{} + m.Stats = &github_com_cortexproject_cortex_pkg_querier_stats.QueryStats{} } if err := m.Stats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err diff --git a/pkg/frontend/v1/frontendv1pb/frontend.proto b/pkg/frontend/v1/frontendv1pb/frontend.proto index 231f918dae..56bc4d5270 100644 --- a/pkg/frontend/v1/frontendv1pb/frontend.proto +++ b/pkg/frontend/v1/frontendv1pb/frontend.proto @@ -39,7 +39,7 @@ message FrontendToClient { message ClientToFrontend { httpgrpc.HTTPResponse httpResponse = 1; string clientID = 2; - stats.Stats stats = 3; + stats.Stats stats = 3[(gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/querier/stats.QueryStats"]; } message NotifyClientShutdownRequest { diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go index f1acd92ad1..b6b88d2a37 100644 --- a/pkg/frontend/v2/frontend_test.go +++ b/pkg/frontend/v2/frontend_test.go @@ -88,7 +88,7 @@ func sendResponseWithDelay(f *Frontend, delay time.Duration, userID string, quer _, _ = f.QueryResult(ctx, &frontendv2pb.QueryResultRequest{ QueryID: queryID, HttpResponse: resp, - Stats: &stats.Stats{}, + Stats: &stats.QueryStats{}, }) } diff --git a/pkg/frontend/v2/frontendv2pb/frontend.pb.go b/pkg/frontend/v2/frontendv2pb/frontend.pb.go index daeb50a8c3..b5ab1a7935 100644 --- a/pkg/frontend/v2/frontendv2pb/frontend.pb.go +++ b/pkg/frontend/v2/frontendv2pb/frontend.pb.go @@ -6,7 +6,8 @@ package frontendv2pb import ( context "context" fmt "fmt" - stats "github.com/cortexproject/cortex/pkg/querier/stats" + _ "github.com/cortexproject/cortex/pkg/querier/stats" + github_com_cortexproject_cortex_pkg_querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" httpgrpc "github.com/weaveworks/common/httpgrpc" @@ -32,9 +33,9 @@ var _ = math.Inf const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type QueryResultRequest struct { - QueryID uint64 `protobuf:"varint,1,opt,name=queryID,proto3" json:"queryID,omitempty"` - HttpResponse *httpgrpc.HTTPResponse `protobuf:"bytes,2,opt,name=httpResponse,proto3" json:"httpResponse,omitempty"` - Stats *stats.Stats `protobuf:"bytes,3,opt,name=stats,proto3" json:"stats,omitempty"` + QueryID uint64 `protobuf:"varint,1,opt,name=queryID,proto3" json:"queryID,omitempty"` + HttpResponse *httpgrpc.HTTPResponse `protobuf:"bytes,2,opt,name=httpResponse,proto3" json:"httpResponse,omitempty"` + Stats *github_com_cortexproject_cortex_pkg_querier_stats.QueryStats `protobuf:"bytes,3,opt,name=stats,proto3,customtype=github.com/cortexproject/cortex/pkg/querier/stats.QueryStats" json:"stats,omitempty"` } func (m *QueryResultRequest) Reset() { *m = QueryResultRequest{} } @@ -83,13 +84,6 @@ func (m *QueryResultRequest) GetHttpResponse() *httpgrpc.HTTPResponse { return nil } -func (m *QueryResultRequest) GetStats() *stats.Stats { - if m != nil { - return m.Stats - } - return nil -} - type QueryResultResponse struct { } @@ -133,29 +127,30 @@ func init() { func init() { proto.RegisterFile("frontend.proto", fileDescriptor_eca3873955a29cfe) } var fileDescriptor_eca3873955a29cfe = []byte{ - // 351 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0xcd, 0x4e, 0x3a, 0x31, - 0x14, 0xc5, 0xdb, 0xff, 0xdf, 0x8f, 0xa4, 0x10, 0x17, 0x35, 0x9a, 0x09, 0x8b, 0x06, 0x67, 0xc5, - 0xc6, 0x69, 0x82, 0xae, 0x4c, 0xdc, 0x10, 0x43, 0x74, 0x27, 0x23, 0x2b, 0x77, 0xcc, 0x58, 0x86, - 0x0f, 0x99, 0x96, 0xb6, 0x03, 0xb2, 0xf3, 0x09, 0x8c, 0x8f, 0xe1, 0xa3, 0xb8, 0x64, 0xc9, 0x52, - 0xca, 0xc6, 0x25, 0x8f, 0x60, 0x68, 0x81, 0x0c, 0x31, 0x71, 0xd3, 0xdc, 0x93, 0x7b, 0x7e, 0xb9, - 0xe7, 0xde, 0xa2, 0xa3, 0xb6, 0xe4, 0xa9, 0x66, 0xe9, 0x53, 0x20, 0x24, 0xd7, 0x1c, 0x17, 0x37, - 0x7a, 0x54, 0x15, 0x51, 0xe9, 0x3c, 0xe9, 0xea, 0x4e, 0x16, 0x05, 0x31, 0x1f, 0xd0, 0x84, 0x27, - 0x9c, 0x5a, 0x53, 0x94, 0xb5, 0xad, 0xb2, 0xc2, 0x56, 0x0e, 0x2e, 0x5d, 0xe6, 0xec, 0x63, 0xd6, - 0x1a, 0xb1, 0x31, 0x97, 0x7d, 0x45, 0x63, 0x3e, 0x18, 0xf0, 0x94, 0x76, 0xb4, 0x16, 0x89, 0x14, - 0xf1, 0xb6, 0x58, 0x53, 0xd7, 0x39, 0x2a, 0xe6, 0x52, 0xb3, 0x17, 0x21, 0x79, 0x8f, 0xc5, 0x7a, - 0xad, 0xa8, 0xe8, 0x27, 0x74, 0x98, 0x31, 0xd9, 0x65, 0x92, 0x2a, 0xdd, 0xd2, 0xca, 0xbd, 0x0e, - 0xf7, 0xdf, 0x20, 0xc2, 0x8d, 0x8c, 0xc9, 0x49, 0xc8, 0x54, 0xf6, 0xac, 0x43, 0x36, 0xcc, 0x98, - 0xd2, 0xd8, 0x43, 0x87, 0x2b, 0x66, 0x72, 0x77, 0xe3, 0xc1, 0x32, 0xac, 0xec, 0x85, 0x1b, 0x89, - 0xaf, 0x50, 0x71, 0x95, 0x20, 0x64, 0x4a, 0xf0, 0x54, 0x31, 0xef, 0x5f, 0x19, 0x56, 0x0a, 0xd5, - 0xd3, 0x60, 0x1b, 0xeb, 0xb6, 0xd9, 0xbc, 0xdf, 0x74, 0xc3, 0x1d, 0x2f, 0xf6, 0xd1, 0xbe, 0x9d, - 0xed, 0xfd, 0xb7, 0x50, 0x31, 0x70, 0x49, 0x1e, 0x56, 0x6f, 0xe8, 0x5a, 0xfe, 0x09, 0x3a, 0xde, - 0xc9, 0xe3, 0xd0, 0x6a, 0x0f, 0xe1, 0xfa, 0xfa, 0xb6, 0x75, 0x2e, 0x1b, 0x6e, 0x1f, 0xdc, 0x44, - 0x85, 0x9c, 0x19, 0x97, 0x83, 0xfc, 0xfd, 0x83, 0xdf, 0x7b, 0x95, 0xce, 0xfe, 0x70, 0xb8, 0x49, - 0x3e, 0xa8, 0xd5, 0xa6, 0x73, 0x02, 0x66, 0x73, 0x02, 0x96, 0x73, 0x02, 0x5f, 0x0d, 0x81, 0x1f, - 0x86, 0xc0, 0x4f, 0x43, 0xe0, 0xd4, 0x10, 0xf8, 0x65, 0x08, 0xfc, 0x36, 0x04, 0x2c, 0x0d, 0x81, - 0xef, 0x0b, 0x02, 0xa6, 0x0b, 0x02, 0x66, 0x0b, 0x02, 0x1e, 0x77, 0xfe, 0x3e, 0x3a, 0xb0, 0xe7, - 0xbd, 0xf8, 0x09, 0x00, 0x00, 0xff, 0xff, 0x02, 0xb0, 0x28, 0xb5, 0x22, 0x02, 0x00, 0x00, + // 367 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0xbd, 0x4e, 0x32, 0x41, + 0x14, 0xdd, 0xf9, 0x7e, 0x93, 0x81, 0x58, 0x8c, 0xd1, 0x6c, 0x28, 0x46, 0xa4, 0xa2, 0x71, 0x27, + 0x41, 0x2b, 0xa3, 0x89, 0x21, 0x86, 0x68, 0x27, 0x2b, 0x95, 0x1d, 0xbb, 0x0c, 0xcb, 0x8f, 0xec, + 0x0c, 0x33, 0xb3, 0x20, 0x9d, 0x8f, 0xe0, 0x63, 0xf8, 0x28, 0x96, 0x5b, 0x12, 0x0b, 0x23, 0x43, + 0x63, 0xc9, 0x23, 0x18, 0x66, 0x80, 0x2c, 0x31, 0x31, 0xb1, 0xb9, 0xb9, 0x27, 0x7b, 0xce, 0xbd, + 0xe7, 0xee, 0x19, 0xb8, 0xd3, 0x16, 0x2c, 0x56, 0x34, 0x6e, 0x79, 0x5c, 0x30, 0xc5, 0x50, 0x7e, + 0x8d, 0x47, 0x15, 0x1e, 0x14, 0x8e, 0xa2, 0xae, 0xea, 0x24, 0x81, 0x17, 0xb2, 0x01, 0x89, 0x58, + 0xc4, 0x88, 0x21, 0x05, 0x49, 0xdb, 0x20, 0x03, 0x4c, 0x67, 0xc5, 0x85, 0x93, 0x0c, 0x7d, 0x4c, + 0x9b, 0x23, 0x3a, 0x66, 0xa2, 0x2f, 0x49, 0xc8, 0x06, 0x03, 0x16, 0x93, 0x8e, 0x52, 0x3c, 0x12, + 0x3c, 0xdc, 0x34, 0x2b, 0xd5, 0x79, 0x46, 0x15, 0x32, 0xa1, 0xe8, 0x03, 0x17, 0xac, 0x47, 0x43, + 0xb5, 0x42, 0x84, 0xf7, 0x23, 0x32, 0x4c, 0xa8, 0xe8, 0x52, 0x41, 0xa4, 0x6a, 0x2a, 0x69, 0xab, + 0x95, 0x97, 0x52, 0x00, 0x51, 0x3d, 0xa1, 0x62, 0xe2, 0x53, 0x99, 0xdc, 0x2b, 0x9f, 0x0e, 0x13, + 0x2a, 0x15, 0x72, 0xe1, 0xff, 0xa5, 0x66, 0x72, 0x7d, 0xe9, 0x82, 0x22, 0x28, 0xff, 0xf1, 0xd7, + 0x10, 0x9d, 0xc2, 0xfc, 0xd2, 0x81, 0x4f, 0x25, 0x67, 0xb1, 0xa4, 0xee, 0xaf, 0x22, 0x28, 0xe7, + 0x2a, 0xfb, 0xde, 0xc6, 0xd6, 0x55, 0xa3, 0x71, 0xb3, 0xfe, 0xea, 0x6f, 0x71, 0x51, 0x0b, 0xfe, + 0x35, 0xbb, 0xdd, 0xdf, 0x46, 0x94, 0xf7, 0xac, 0x93, 0xdb, 0x65, 0xad, 0x5e, 0xbc, 0xbe, 0x1d, + 0x9c, 0xfd, 0xf8, 0x18, 0xcf, 0x98, 0x37, 0x13, 0x7c, 0x3b, 0xbc, 0xb4, 0x07, 0x77, 0xb7, 0x2e, + 0xb2, 0xcb, 0x2b, 0x3d, 0x88, 0x6a, 0xab, 0x74, 0x6a, 0x4c, 0xd4, 0xed, 0x10, 0xd4, 0x80, 0xb9, + 0x0c, 0x19, 0x15, 0xbd, 0x6c, 0x82, 0xde, 0xd7, 0x3f, 0x53, 0x38, 0xfc, 0x86, 0x61, 0x37, 0x95, + 0x9c, 0x6a, 0x35, 0x9d, 0x61, 0x67, 0x3a, 0xc3, 0xce, 0x62, 0x86, 0xc1, 0xa3, 0xc6, 0xe0, 0x59, + 0x63, 0xf0, 0xa2, 0x31, 0x48, 0x35, 0x06, 0xef, 0x1a, 0x83, 0x0f, 0x8d, 0x9d, 0x85, 0xc6, 0xe0, + 0x69, 0x8e, 0x9d, 0x74, 0x8e, 0x9d, 0xe9, 0x1c, 0x3b, 0x77, 0x5b, 0xaf, 0x27, 0xf8, 0x67, 0x02, + 0x3a, 0xfe, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x31, 0xd0, 0x9a, 0x67, 0x64, 0x02, 0x00, 0x00, } func (this *QueryResultRequest) Equal(that interface{}) bool { @@ -183,7 +178,11 @@ func (this *QueryResultRequest) Equal(that interface{}) bool { if !this.HttpResponse.Equal(that1.HttpResponse) { return false } - if !this.Stats.Equal(that1.Stats) { + if that1.Stats == nil { + if this.Stats != nil { + return false + } + } else if !this.Stats.Equal(*that1.Stats) { return false } return true @@ -219,9 +218,7 @@ func (this *QueryResultRequest) GoString() string { if this.HttpResponse != nil { s = append(s, "HttpResponse: "+fmt.Sprintf("%#v", this.HttpResponse)+",\n") } - if this.Stats != nil { - s = append(s, "Stats: "+fmt.Sprintf("%#v", this.Stats)+",\n") - } + s = append(s, "Stats: "+fmt.Sprintf("%#v", this.Stats)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -345,11 +342,11 @@ func (m *QueryResultRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = l if m.Stats != nil { { - size, err := m.Stats.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { + size := m.Stats.Size() + i -= size + if _, err := m.Stats.MarshalTo(dAtA[i:]); err != nil { return 0, err } - i -= size i = encodeVarintFrontend(dAtA, i, uint64(size)) } i-- @@ -451,7 +448,7 @@ func (this *QueryResultRequest) String() string { s := strings.Join([]string{`&QueryResultRequest{`, `QueryID:` + fmt.Sprintf("%v", this.QueryID) + `,`, `HttpResponse:` + strings.Replace(fmt.Sprintf("%v", this.HttpResponse), "HTTPResponse", "httpgrpc.HTTPResponse", 1) + `,`, - `Stats:` + strings.Replace(fmt.Sprintf("%v", this.Stats), "Stats", "stats.Stats", 1) + `,`, + `Stats:` + fmt.Sprintf("%v", this.Stats) + `,`, `}`, }, "") return s @@ -587,7 +584,7 @@ func (m *QueryResultRequest) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Stats == nil { - m.Stats = &stats.Stats{} + m.Stats = &github_com_cortexproject_cortex_pkg_querier_stats.QueryStats{} } if err := m.Stats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err diff --git a/pkg/frontend/v2/frontendv2pb/frontend.proto b/pkg/frontend/v2/frontendv2pb/frontend.proto index b93106d787..555c249d40 100644 --- a/pkg/frontend/v2/frontendv2pb/frontend.proto +++ b/pkg/frontend/v2/frontendv2pb/frontend.proto @@ -19,7 +19,7 @@ service FrontendForQuerier { message QueryResultRequest { uint64 queryID = 1; httpgrpc.HTTPResponse httpResponse = 2; - stats.Stats stats = 3; + stats.Stats stats = 3[(gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/querier/stats.QueryStats"]; // There is no userID field here, because Querier puts userID into the context when // calling QueryResult, and that is where Frontend expects to find it. diff --git a/pkg/querier/stats/stats.go b/pkg/querier/stats/stats.go index 57c0fc23f5..d5c71f86ed 100644 --- a/pkg/querier/stats/stats.go +++ b/pkg/querier/stats/stats.go @@ -2,6 +2,8 @@ package stats import ( "context" + "fmt" + "sync" "sync/atomic" //lint:ignore faillint we can't use go.uber.org/atomic with a protobuf struct without wrapping it. "time" @@ -12,21 +14,26 @@ type contextKey int var ctxKey = contextKey(0) +type QueryStats struct { + Stats + m sync.Mutex +} + // ContextWithEmptyStats returns a context with empty stats. -func ContextWithEmptyStats(ctx context.Context) (*Stats, context.Context) { - stats := &Stats{} +func ContextWithEmptyStats(ctx context.Context) (*QueryStats, context.Context) { + stats := &QueryStats{} ctx = context.WithValue(ctx, ctxKey, stats) return stats, ctx } // FromContext gets the Stats out of the Context. Returns nil if stats have not // been initialised in the context. -func FromContext(ctx context.Context) *Stats { +func FromContext(ctx context.Context) *QueryStats { o := ctx.Value(ctxKey) if o == nil { return nil } - return o.(*Stats) + return o.(*QueryStats) } // IsEnabled returns whether stats tracking is enabled in the context. @@ -37,7 +44,7 @@ func IsEnabled(ctx context.Context) bool { } // AddWallTime adds some time to the counter. -func (s *Stats) AddWallTime(t time.Duration) { +func (s *QueryStats) AddWallTime(t time.Duration) { if s == nil { return } @@ -46,7 +53,7 @@ func (s *Stats) AddWallTime(t time.Duration) { } // LoadWallTime returns current wall time. -func (s *Stats) LoadWallTime() time.Duration { +func (s *QueryStats) LoadWallTime() time.Duration { if s == nil { return 0 } @@ -54,7 +61,7 @@ func (s *Stats) LoadWallTime() time.Duration { return time.Duration(atomic.LoadInt64((*int64)(&s.WallTime))) } -func (s *Stats) AddFetchedSeries(series uint64) { +func (s *QueryStats) AddFetchedSeries(series uint64) { if s == nil { return } @@ -62,7 +69,46 @@ func (s *Stats) AddFetchedSeries(series uint64) { atomic.AddUint64(&s.FetchedSeriesCount, series) } -func (s *Stats) LoadFetchedSeries() uint64 { +func (s *QueryStats) AddExtraFields(fieldsVals ...interface{}) { + if s == nil { + return + } + + s.m.Lock() + defer s.m.Unlock() + + if s.ExtraFields == nil { + s.ExtraFields = map[string]string{} + } + + if len(fieldsVals)%2 == 1 { + fieldsVals = append(fieldsVals, "") + } + + for i := 0; i < len(fieldsVals); i += 2 { + if v, ok := fieldsVals[i].(string); ok { + s.ExtraFields[v] = fmt.Sprintf("%v", fieldsVals[i+1]) + } + } +} + +func (s *QueryStats) LoadExtraFields() []interface{} { + if s == nil { + return []interface{}{} + } + + s.m.Lock() + defer s.m.Unlock() + + r := make([]interface{}, 0, len(s.ExtraFields)) + for k, v := range s.ExtraFields { + r = append(r, k, v) + } + + return r +} + +func (s *QueryStats) LoadFetchedSeries() uint64 { if s == nil { return 0 } @@ -70,7 +116,7 @@ func (s *Stats) LoadFetchedSeries() uint64 { return atomic.LoadUint64(&s.FetchedSeriesCount) } -func (s *Stats) AddFetchedChunkBytes(bytes uint64) { +func (s *QueryStats) AddFetchedChunkBytes(bytes uint64) { if s == nil { return } @@ -78,7 +124,7 @@ func (s *Stats) AddFetchedChunkBytes(bytes uint64) { atomic.AddUint64(&s.FetchedChunkBytes, bytes) } -func (s *Stats) LoadFetchedChunkBytes() uint64 { +func (s *QueryStats) LoadFetchedChunkBytes() uint64 { if s == nil { return 0 } @@ -86,7 +132,7 @@ func (s *Stats) LoadFetchedChunkBytes() uint64 { return atomic.LoadUint64(&s.FetchedChunkBytes) } -func (s *Stats) AddFetchedDataBytes(bytes uint64) { +func (s *QueryStats) AddFetchedDataBytes(bytes uint64) { if s == nil { return } @@ -94,7 +140,7 @@ func (s *Stats) AddFetchedDataBytes(bytes uint64) { atomic.AddUint64(&s.FetchedDataBytes, bytes) } -func (s *Stats) LoadFetchedDataBytes() uint64 { +func (s *QueryStats) LoadFetchedDataBytes() uint64 { if s == nil { return 0 } @@ -103,7 +149,7 @@ func (s *Stats) LoadFetchedDataBytes() uint64 { } // Merge the provide Stats into this one. -func (s *Stats) Merge(other *Stats) { +func (s *QueryStats) Merge(other *QueryStats) { if s == nil || other == nil { return } @@ -112,6 +158,7 @@ func (s *Stats) Merge(other *Stats) { s.AddFetchedSeries(other.LoadFetchedSeries()) s.AddFetchedChunkBytes(other.LoadFetchedChunkBytes()) s.AddFetchedDataBytes(other.LoadFetchedDataBytes()) + s.AddExtraFields(other.LoadExtraFields()...) } func ShouldTrackHTTPGRPCResponse(r *httpgrpc.HTTPResponse) bool { diff --git a/pkg/querier/stats/stats.pb.go b/pkg/querier/stats/stats.pb.go index 0f6ad5377b..d0a0653cfe 100644 --- a/pkg/querier/stats/stats.pb.go +++ b/pkg/querier/stats/stats.pb.go @@ -7,6 +7,8 @@ import ( fmt "fmt" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" + github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" + _ "github.com/gogo/protobuf/types" github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" _ "github.com/golang/protobuf/ptypes/duration" io "io" @@ -38,6 +40,8 @@ type Stats struct { FetchedChunkBytes uint64 `protobuf:"varint,3,opt,name=fetched_chunk_bytes,json=fetchedChunkBytes,proto3" json:"fetched_chunk_bytes,omitempty"` // The number of bytes of data fetched for the query FetchedDataBytes uint64 `protobuf:"varint,4,opt,name=fetched_data_bytes,json=fetchedDataBytes,proto3" json:"fetched_data_bytes,omitempty"` + // Extra fields to be reported on the stats log + ExtraFields map[string]string `protobuf:"bytes,5,rep,name=extra_fields,json=extraFields,proto3" json:"extra_fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *Stats) Reset() { *m = Stats{} } @@ -100,33 +104,47 @@ func (m *Stats) GetFetchedDataBytes() uint64 { return 0 } +func (m *Stats) GetExtraFields() map[string]string { + if m != nil { + return m.ExtraFields + } + return nil +} + func init() { proto.RegisterType((*Stats)(nil), "stats.Stats") + proto.RegisterMapType((map[string]string)(nil), "stats.Stats.ExtraFieldsEntry") } func init() { proto.RegisterFile("stats.proto", fileDescriptor_b4756a0aec8b9d44) } var fileDescriptor_b4756a0aec8b9d44 = []byte{ - // 300 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x90, 0x31, 0x4e, 0xc3, 0x30, - 0x14, 0x86, 0xfd, 0xa0, 0x45, 0xc5, 0x5d, 0xc0, 0x30, 0x94, 0x0e, 0xaf, 0x15, 0x53, 0x07, 0x70, - 0x11, 0x8c, 0x2c, 0xa8, 0xed, 0x09, 0x5a, 0x26, 0x96, 0xc8, 0x49, 0xdd, 0x34, 0xa2, 0x8d, 0x51, - 0xe2, 0x08, 0xb1, 0x71, 0x04, 0x46, 0x8e, 0xc0, 0x51, 0x3a, 0x66, 0x2c, 0x0b, 0x10, 0x67, 0x61, - 0xec, 0x11, 0x50, 0x9c, 0x44, 0x6c, 0xfe, 0xf5, 0xfd, 0xdf, 0x2f, 0xf9, 0xd1, 0x76, 0xac, 0x85, - 0x8e, 0xf9, 0x53, 0xa4, 0xb4, 0x62, 0x4d, 0x1b, 0xba, 0x97, 0x7e, 0xa0, 0x97, 0x89, 0xcb, 0x3d, - 0xb5, 0x1e, 0xfa, 0xca, 0x57, 0x43, 0x4b, 0xdd, 0x64, 0x61, 0x93, 0x0d, 0xf6, 0x55, 0x5a, 0x5d, - 0xf4, 0x95, 0xf2, 0x57, 0xf2, 0xbf, 0x35, 0x4f, 0x22, 0xa1, 0x03, 0x15, 0x96, 0xfc, 0xfc, 0x13, - 0x68, 0x73, 0x56, 0x0c, 0xb3, 0x3b, 0x7a, 0xf8, 0x2c, 0x56, 0x2b, 0x47, 0x07, 0x6b, 0xd9, 0x81, - 0x3e, 0x0c, 0xda, 0xd7, 0x67, 0xbc, 0xb4, 0x79, 0x6d, 0xf3, 0x49, 0x65, 0x8f, 0x5a, 0x9b, 0xaf, - 0x1e, 0x79, 0xff, 0xee, 0xc1, 0xb4, 0x55, 0x58, 0xf7, 0xc1, 0x5a, 0xb2, 0x2b, 0x7a, 0xba, 0x90, - 0xda, 0x5b, 0xca, 0xb9, 0x13, 0xcb, 0x28, 0x90, 0xb1, 0xe3, 0xa9, 0x24, 0xd4, 0x9d, 0xbd, 0x3e, - 0x0c, 0x1a, 0x53, 0x56, 0xb1, 0x99, 0x45, 0xe3, 0x82, 0x30, 0x4e, 0x4f, 0x6a, 0xc3, 0x5b, 0x26, - 0xe1, 0xa3, 0xe3, 0xbe, 0x68, 0x19, 0x77, 0xf6, 0xad, 0x70, 0x5c, 0xa1, 0x71, 0x41, 0x46, 0x05, - 0x60, 0x17, 0xb4, 0x5e, 0x71, 0xe6, 0x42, 0x8b, 0xaa, 0xde, 0xb0, 0xf5, 0xa3, 0x8a, 0x4c, 0x84, - 0x16, 0xb6, 0x3d, 0xba, 0x4d, 0x33, 0x24, 0xdb, 0x0c, 0xc9, 0x2e, 0x43, 0x78, 0x35, 0x08, 0x1f, - 0x06, 0x61, 0x63, 0x10, 0x52, 0x83, 0xf0, 0x63, 0x10, 0x7e, 0x0d, 0x92, 0x9d, 0x41, 0x78, 0xcb, - 0x91, 0xa4, 0x39, 0x92, 0x6d, 0x8e, 0xe4, 0xa1, 0xbc, 0xb3, 0x7b, 0x60, 0xff, 0x7c, 0xf3, 0x17, - 0x00, 0x00, 0xff, 0xff, 0xb7, 0x24, 0x3a, 0xa7, 0x84, 0x01, 0x00, 0x00, + // 388 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0x3f, 0x6e, 0xe2, 0x40, + 0x18, 0xc5, 0x3d, 0x80, 0x57, 0x30, 0xde, 0x82, 0x9d, 0xa5, 0x30, 0x48, 0x3b, 0xa0, 0xad, 0x28, + 0x76, 0x87, 0x15, 0xdb, 0x44, 0x89, 0x14, 0x21, 0xfe, 0xe4, 0x00, 0x26, 0x55, 0x1a, 0x6b, 0x6c, + 0x0f, 0xc6, 0xc2, 0x78, 0x22, 0x7b, 0x9c, 0xc4, 0x5d, 0x8e, 0x90, 0x32, 0x47, 0xc8, 0x51, 0x28, + 0x29, 0x91, 0x22, 0x25, 0xc1, 0x34, 0x29, 0x39, 0x42, 0xe4, 0xb1, 0x11, 0x12, 0xdd, 0xf7, 0xfc, + 0x7b, 0xef, 0xb3, 0xbe, 0xa7, 0x81, 0x5a, 0x24, 0xa8, 0x88, 0xc8, 0x6d, 0xc8, 0x05, 0x47, 0xaa, + 0x14, 0xad, 0xbf, 0xae, 0x27, 0xe6, 0xb1, 0x45, 0x6c, 0xbe, 0xec, 0xb9, 0xdc, 0xe5, 0x3d, 0x49, + 0xad, 0x78, 0x26, 0x95, 0x14, 0x72, 0xca, 0x53, 0x2d, 0xec, 0x72, 0xee, 0xfa, 0xec, 0xe8, 0x72, + 0xe2, 0x90, 0x0a, 0x8f, 0x07, 0x05, 0x6f, 0x9e, 0x72, 0x1a, 0x24, 0x39, 0xfa, 0xfd, 0x5a, 0x82, + 0xea, 0x34, 0xfb, 0x27, 0x1a, 0xc0, 0xda, 0x3d, 0xf5, 0x7d, 0x53, 0x78, 0x4b, 0xa6, 0x83, 0x0e, + 0xe8, 0x6a, 0xfd, 0x26, 0xc9, 0x83, 0xe4, 0x10, 0x24, 0xe3, 0x62, 0xf1, 0xb0, 0xba, 0x7a, 0x6b, + 0x2b, 0xcf, 0xef, 0x6d, 0x60, 0x54, 0xb3, 0xd4, 0xb5, 0xb7, 0x64, 0xe8, 0x1f, 0x6c, 0xcc, 0x98, + 0xb0, 0xe7, 0xcc, 0x31, 0x23, 0x16, 0x7a, 0x2c, 0x32, 0x6d, 0x1e, 0x07, 0x42, 0x2f, 0x75, 0x40, + 0xb7, 0x62, 0xa0, 0x82, 0x4d, 0x25, 0x1a, 0x65, 0x04, 0x11, 0xf8, 0xf3, 0x90, 0xb0, 0xe7, 0x71, + 0xb0, 0x30, 0xad, 0x44, 0xb0, 0x48, 0x2f, 0xcb, 0xc0, 0x8f, 0x02, 0x8d, 0x32, 0x32, 0xcc, 0x00, + 0xfa, 0x03, 0x0f, 0x5b, 0x4c, 0x87, 0x0a, 0x5a, 0xd8, 0x2b, 0xd2, 0x5e, 0x2f, 0xc8, 0x98, 0x0a, + 0x9a, 0xbb, 0x07, 0xf0, 0x3b, 0x7b, 0x10, 0x21, 0x35, 0x67, 0x1e, 0xf3, 0x9d, 0x48, 0x57, 0x3b, + 0xe5, 0xae, 0xd6, 0xff, 0x45, 0xf2, 0xc2, 0xe5, 0xd5, 0x64, 0x92, 0x19, 0xae, 0x24, 0x9f, 0x04, + 0x22, 0x4c, 0x0c, 0x8d, 0x1d, 0xbf, 0xb4, 0x2e, 0x61, 0xfd, 0xd4, 0x80, 0xea, 0xb0, 0xbc, 0x60, + 0x89, 0x6c, 0xa8, 0x66, 0x64, 0x23, 0x6a, 0x40, 0xf5, 0x8e, 0xfa, 0x31, 0x93, 0x87, 0xd6, 0x8c, + 0x5c, 0x9c, 0x97, 0xce, 0xc0, 0xf0, 0x62, 0xbd, 0xc5, 0xca, 0x66, 0x8b, 0x95, 0xfd, 0x16, 0x83, + 0xc7, 0x14, 0x83, 0x97, 0x14, 0x83, 0x55, 0x8a, 0xc1, 0x3a, 0xc5, 0xe0, 0x23, 0xc5, 0xe0, 0x33, + 0xc5, 0xca, 0x3e, 0xc5, 0xe0, 0x69, 0x87, 0x95, 0xf5, 0x0e, 0x2b, 0x9b, 0x1d, 0x56, 0x6e, 0xf2, + 0x47, 0x60, 0x7d, 0x93, 0xad, 0xff, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x25, 0x6d, 0xec, 0xdd, + 0x21, 0x02, 0x00, 0x00, } func (this *Stats) Equal(that interface{}) bool { @@ -160,18 +178,39 @@ func (this *Stats) Equal(that interface{}) bool { if this.FetchedDataBytes != that1.FetchedDataBytes { return false } + if len(this.ExtraFields) != len(that1.ExtraFields) { + return false + } + for i := range this.ExtraFields { + if this.ExtraFields[i] != that1.ExtraFields[i] { + return false + } + } return true } func (this *Stats) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 8) + s := make([]string, 0, 9) s = append(s, "&stats.Stats{") s = append(s, "WallTime: "+fmt.Sprintf("%#v", this.WallTime)+",\n") s = append(s, "FetchedSeriesCount: "+fmt.Sprintf("%#v", this.FetchedSeriesCount)+",\n") s = append(s, "FetchedChunkBytes: "+fmt.Sprintf("%#v", this.FetchedChunkBytes)+",\n") s = append(s, "FetchedDataBytes: "+fmt.Sprintf("%#v", this.FetchedDataBytes)+",\n") + keysForExtraFields := make([]string, 0, len(this.ExtraFields)) + for k, _ := range this.ExtraFields { + keysForExtraFields = append(keysForExtraFields, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForExtraFields) + mapStringForExtraFields := "map[string]string{" + for _, k := range keysForExtraFields { + mapStringForExtraFields += fmt.Sprintf("%#v: %#v,", k, this.ExtraFields[k]) + } + mapStringForExtraFields += "}" + if this.ExtraFields != nil { + s = append(s, "ExtraFields: "+mapStringForExtraFields+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -203,6 +242,25 @@ func (m *Stats) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.ExtraFields) > 0 { + for k := range m.ExtraFields { + v := m.ExtraFields[k] + baseI := i + i -= len(v) + copy(dAtA[i:], v) + i = encodeVarintStats(dAtA, i, uint64(len(v))) + i-- + dAtA[i] = 0x12 + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintStats(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintStats(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x2a + } + } if m.FetchedDataBytes != 0 { i = encodeVarintStats(dAtA, i, uint64(m.FetchedDataBytes)) i-- @@ -257,6 +315,14 @@ func (m *Stats) Size() (n int) { if m.FetchedDataBytes != 0 { n += 1 + sovStats(uint64(m.FetchedDataBytes)) } + if len(m.ExtraFields) > 0 { + for k, v := range m.ExtraFields { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovStats(uint64(len(k))) + 1 + len(v) + sovStats(uint64(len(v))) + n += mapEntrySize + 1 + sovStats(uint64(mapEntrySize)) + } + } return n } @@ -270,11 +336,22 @@ func (this *Stats) String() string { if this == nil { return "nil" } + keysForExtraFields := make([]string, 0, len(this.ExtraFields)) + for k, _ := range this.ExtraFields { + keysForExtraFields = append(keysForExtraFields, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForExtraFields) + mapStringForExtraFields := "map[string]string{" + for _, k := range keysForExtraFields { + mapStringForExtraFields += fmt.Sprintf("%v: %v,", k, this.ExtraFields[k]) + } + mapStringForExtraFields += "}" s := strings.Join([]string{`&Stats{`, `WallTime:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.WallTime), "Duration", "duration.Duration", 1), `&`, ``, 1) + `,`, `FetchedSeriesCount:` + fmt.Sprintf("%v", this.FetchedSeriesCount) + `,`, `FetchedChunkBytes:` + fmt.Sprintf("%v", this.FetchedChunkBytes) + `,`, `FetchedDataBytes:` + fmt.Sprintf("%v", this.FetchedDataBytes) + `,`, + `ExtraFields:` + mapStringForExtraFields + `,`, `}`, }, "") return s @@ -406,6 +483,133 @@ func (m *Stats) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ExtraFields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStats + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStats + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ExtraFields == nil { + m.ExtraFields = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthStats + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthStats + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthStats + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue < 0 { + return ErrInvalidLengthStats + } + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := skipStats(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStats + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.ExtraFields[mapkey] = mapvalue + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) diff --git a/pkg/querier/stats/stats.proto b/pkg/querier/stats/stats.proto index 4cbc4e5b7e..c0e509d9a1 100644 --- a/pkg/querier/stats/stats.proto +++ b/pkg/querier/stats/stats.proto @@ -6,6 +6,7 @@ option go_package = "stats"; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/any.proto"; option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; @@ -19,4 +20,6 @@ message Stats { uint64 fetched_chunk_bytes = 3; // The number of bytes of data fetched for the query uint64 fetched_data_bytes = 4; + // Extra fields to be reported on the stats log + map extra_fields = 5; } diff --git a/pkg/querier/stats/stats_test.go b/pkg/querier/stats/stats_test.go index add83dfaf5..41daec7bb6 100644 --- a/pkg/querier/stats/stats_test.go +++ b/pkg/querier/stats/stats_test.go @@ -18,7 +18,7 @@ func TestStats_WallTime(t *testing.T) { }) t.Run("add and load wall time nil receiver", func(t *testing.T) { - var stats *Stats + var stats *QueryStats stats.AddWallTime(time.Second) assert.Equal(t, time.Duration(0), stats.LoadWallTime()) @@ -35,13 +35,30 @@ func TestStats_AddFetchedSeries(t *testing.T) { }) t.Run("add and load series nil receiver", func(t *testing.T) { - var stats *Stats + var stats *QueryStats stats.AddFetchedSeries(50) assert.Equal(t, uint64(0), stats.LoadFetchedSeries()) }) } +func TestQueryStats_AddExtraFields(t *testing.T) { + t.Run("add and load extra fields", func(t *testing.T) { + stats, _ := ContextWithEmptyStats(context.Background()) + stats.AddExtraFields("a", "b") + stats.AddExtraFields("c") + + checkExtraFields(t, []interface{}{"a", "b", "c", ""}, stats.LoadExtraFields()) + }) + + t.Run("add and load extra fields nil receiver", func(t *testing.T) { + var stats *QueryStats + stats.AddExtraFields("a", "b") + + checkExtraFields(t, []interface{}{}, stats.LoadExtraFields()) + }) +} + func TestStats_AddFetchedChunkBytes(t *testing.T) { t.Run("add and load bytes", func(t *testing.T) { stats, _ := ContextWithEmptyStats(context.Background()) @@ -52,7 +69,7 @@ func TestStats_AddFetchedChunkBytes(t *testing.T) { }) t.Run("add and load bytes nil receiver", func(t *testing.T) { - var stats *Stats + var stats *QueryStats stats.AddFetchedChunkBytes(1024) assert.Equal(t, uint64(0), stats.LoadFetchedChunkBytes()) @@ -69,7 +86,7 @@ func TestStats_AddFetchedDataBytes(t *testing.T) { }) t.Run("add and load bytes nil receiver", func(t *testing.T) { - var stats *Stats + var stats *QueryStats stats.AddFetchedDataBytes(1024) assert.Equal(t, uint64(0), stats.LoadFetchedDataBytes()) @@ -78,17 +95,20 @@ func TestStats_AddFetchedDataBytes(t *testing.T) { func TestStats_Merge(t *testing.T) { t.Run("merge two stats objects", func(t *testing.T) { - stats1 := &Stats{} + stats1 := &QueryStats{} stats1.AddWallTime(time.Millisecond) stats1.AddFetchedSeries(50) stats1.AddFetchedChunkBytes(42) stats1.AddFetchedDataBytes(100) + stats1.AddExtraFields("a", "b") + stats1.AddExtraFields("a", "b") - stats2 := &Stats{} + stats2 := &QueryStats{} stats2.AddWallTime(time.Second) stats2.AddFetchedSeries(60) stats2.AddFetchedChunkBytes(100) stats2.AddFetchedDataBytes(101) + stats2.AddExtraFields("c", "d") stats1.Merge(stats2) @@ -96,11 +116,12 @@ func TestStats_Merge(t *testing.T) { assert.Equal(t, uint64(110), stats1.LoadFetchedSeries()) assert.Equal(t, uint64(142), stats1.LoadFetchedChunkBytes()) assert.Equal(t, uint64(201), stats1.LoadFetchedDataBytes()) + checkExtraFields(t, []interface{}{"a", "b", "c", "d"}, stats1.LoadExtraFields()) }) t.Run("merge two nil stats objects", func(t *testing.T) { - var stats1 *Stats - var stats2 *Stats + var stats1 *QueryStats + var stats2 *QueryStats stats1.Merge(stats2) @@ -108,5 +129,19 @@ func TestStats_Merge(t *testing.T) { assert.Equal(t, uint64(0), stats1.LoadFetchedSeries()) assert.Equal(t, uint64(0), stats1.LoadFetchedChunkBytes()) assert.Equal(t, uint64(0), stats1.LoadFetchedDataBytes()) + checkExtraFields(t, []interface{}{}, stats1.LoadExtraFields()) }) } + +func checkExtraFields(t *testing.T, expected, actual []interface{}) { + assert.Equal(t, len(expected), len(actual)) + expectedMap := map[string]string{} + actualMap := map[string]string{} + + for i := 0; i < len(expected); i += 2 { + expectedMap[expected[i].(string)] = expected[i+1].(string) + actualMap[actual[i].(string)] = actual[i+1].(string) + } + + assert.Equal(t, expectedMap, actualMap) +} diff --git a/pkg/querier/tripperware/shard_by.go b/pkg/querier/tripperware/shard_by.go index 67ee8c97ca..8bb218af32 100644 --- a/pkg/querier/tripperware/shard_by.go +++ b/pkg/querier/tripperware/shard_by.go @@ -10,6 +10,7 @@ import ( "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/weaveworks/common/httpgrpc" + querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" cquerysharding "github.com/cortexproject/cortex/pkg/querysharding" "github.com/cortexproject/cortex/pkg/tenant" util_log "github.com/cortexproject/cortex/pkg/util/log" @@ -37,6 +38,7 @@ type shardBy struct { func (s shardBy) Do(ctx context.Context, r Request) (Response, error) { tenantIDs, err := tenant.TenantIDs(ctx) + stats := querier_stats.FromContext(ctx) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) @@ -54,6 +56,12 @@ func (s shardBy) Do(ctx context.Context, r Request) (Response, error) { level.Warn(logger).Log("msg", "error analyzing query", "q", r.GetQuery(), "err", err) } + stats.AddExtraFields( + "shard_by.is_shardable", analysis.IsShardable(), + "shard_by.num_shards", numShards, + "shard_by.sharding_labels", analysis.ShardingLabels(), + ) + if err != nil || !analysis.IsShardable() { return s.next.Do(ctx, r) } diff --git a/pkg/querier/worker/frontend_processor.go b/pkg/querier/worker/frontend_processor.go index 9189ff1dd4..6120817666 100644 --- a/pkg/querier/worker/frontend_processor.go +++ b/pkg/querier/worker/frontend_processor.go @@ -95,7 +95,7 @@ func (fp *frontendProcessor) process(c frontendv1pb.Frontend_ProcessClient) erro // and cancel the query. We don't actually handle queries in parallel // here, as we're running in lock step with the server - each Recv is // paired with a Send. - go fp.runRequest(ctx, request.HttpRequest, request.StatsEnabled, func(response *httpgrpc.HTTPResponse, stats *stats.Stats) error { + go fp.runRequest(ctx, request.HttpRequest, request.StatsEnabled, func(response *httpgrpc.HTTPResponse, stats *stats.QueryStats) error { return c.Send(&frontendv1pb.ClientToFrontend{ HttpResponse: response, Stats: stats, @@ -114,8 +114,8 @@ func (fp *frontendProcessor) process(c frontendv1pb.Frontend_ProcessClient) erro } } -func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.HTTPRequest, statsEnabled bool, sendHTTPResponse func(response *httpgrpc.HTTPResponse, stats *stats.Stats) error) { - var stats *querier_stats.Stats +func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.HTTPRequest, statsEnabled bool, sendHTTPResponse func(response *httpgrpc.HTTPResponse, stats *stats.QueryStats) error) { + var stats *querier_stats.QueryStats if statsEnabled { stats, ctx = querier_stats.ContextWithEmptyStats(ctx) } diff --git a/pkg/querier/worker/scheduler_processor.go b/pkg/querier/worker/scheduler_processor.go index ec9b52e6da..379c0bd8ec 100644 --- a/pkg/querier/worker/scheduler_processor.go +++ b/pkg/querier/worker/scheduler_processor.go @@ -152,7 +152,7 @@ func (sp *schedulerProcessor) querierLoop(c schedulerpb.SchedulerForQuerier_Quer } func (sp *schedulerProcessor) runRequest(ctx context.Context, logger log.Logger, queryID uint64, frontendAddress string, statsEnabled bool, request *httpgrpc.HTTPRequest) { - var stats *querier_stats.Stats + var stats *querier_stats.QueryStats if statsEnabled { stats, ctx = querier_stats.ContextWithEmptyStats(ctx) }