From e4ad4e0950a9711ddc81a6cf75df1c2ec211d3c2 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 17 Jun 2019 21:32:11 -0400 Subject: [PATCH 01/25] setup consul --- production/helm/loki-distributed/.helmignore | 22 ++ production/helm/loki-distributed/Chart.yaml | 13 ++ .../loki-distributed/templates/_helpers.tpl | 32 +++ .../templates/consul/configmap.yaml | 217 ++++++++++++++++++ .../templates/consul/deployment.yaml | 127 ++++++++++ .../templates/consul/service.yaml | 41 ++++ production/helm/loki-distributed/values.yaml | 176 ++++++++++++++ 7 files changed, 628 insertions(+) create mode 100644 production/helm/loki-distributed/.helmignore create mode 100644 production/helm/loki-distributed/Chart.yaml create mode 100644 production/helm/loki-distributed/templates/_helpers.tpl create mode 100644 production/helm/loki-distributed/templates/consul/configmap.yaml create mode 100644 production/helm/loki-distributed/templates/consul/deployment.yaml create mode 100644 production/helm/loki-distributed/templates/consul/service.yaml create mode 100644 production/helm/loki-distributed/values.yaml diff --git a/production/helm/loki-distributed/.helmignore b/production/helm/loki-distributed/.helmignore new file mode 100644 index 000000000000..50af03172541 --- /dev/null +++ b/production/helm/loki-distributed/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/production/helm/loki-distributed/Chart.yaml b/production/helm/loki-distributed/Chart.yaml new file mode 100644 index 000000000000..8b003d090d11 --- /dev/null +++ b/production/helm/loki-distributed/Chart.yaml @@ -0,0 +1,13 @@ +name: loki-distributed +version: 0.1.0 +appVersion: 0.0.1 +kubeVersion: "^1.10.0-0" +description: "Loki: like Prometheus, but for logs." +home: https://grafana.com/loki +icon: https://github.com/grafana/loki/raw/master/docs/logo.png +sources: + - https://github.com/grafana/loki +maintainers: + - name: Loki Maintainers + email: lokiproject@googlegroups.com +engine: gotpl diff --git a/production/helm/loki-distributed/templates/_helpers.tpl b/production/helm/loki-distributed/templates/_helpers.tpl new file mode 100644 index 000000000000..90d9de08e2b5 --- /dev/null +++ b/production/helm/loki-distributed/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "loki.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "loki.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "loki.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/production/helm/loki-distributed/templates/consul/configmap.yaml b/production/helm/loki-distributed/templates/consul/configmap.yaml new file mode 100644 index 000000000000..dcd3800fc34a --- /dev/null +++ b/production/helm/loki-distributed/templates/consul/configmap.yaml @@ -0,0 +1,217 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "loki.fullname" . }}-consul-config + labels: + name: consul-config + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + consul-config.json: "{\"leave_on_terminate\": true, \"telemetry\": {\"dogstatsd_addr\": \"127.0.0.1:9125\"}}\n" + consul.config: "{\"leave_on_terminate\": true, \"telemetry\": {\"dogstatsd_addr\": \"127.0.0.1:9125\"}}" + mapping: | + mappings: + - match: consul.*.runtime.* + name: consul_runtime + labels: + type: $2 + - match: consul.runtime.total_gc_pause_ns + name: consul_runtime_total_gc_pause_ns + labels: + type: $2 + - match: consul.consul.health.service.query-tag.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3 + - match: consul.consul.health.service.query-tag.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4 + - match: consul.consul.health.service.query-tag.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10.$11 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10.$11.$12 + - match: consul.consul.catalog.deregister + name: consul_catalog_deregister + labels: {} + - match: consul.consul.dns.domain_query.*.*.*.*.* + name: consul_dns_domain_query + labels: + query: $1.$2.$3.$4.$5 + - match: consul.consul.health.service.not-found.* + name: consul_health_service_not_found + labels: + query: $1 + - match: consul.consul.health.service.query.* + name: consul_health_service_query + labels: + query: $1 + - match: consul.*.memberlist.health.score + name: consul_memberlist_health_score + labels: {} + - match: consul.serf.queue.* + name: consul_serf_events + labels: + type: $1 + - match: consul.serf.snapshot.appendLine + name: consul_serf_snapshot_appendLine + labels: + type: $1 + - match: consul.serf.coordinate.adjustment-ms + name: consul_serf_coordinate_adjustment_ms + labels: {} + - match: consul.consul.rpc.query + name: consul_rpc_query + labels: {} + - match: consul.*.consul.session_ttl.active + name: consul_session_ttl_active + labels: {} + - match: consul.raft.rpc.* + name: consul_raft_rpc + labels: + type: $1 + - match: consul.raft.rpc.appendEntries.storeLogs + name: consul_raft_rpc_appendEntries_storeLogs + labels: + type: $1 + - match: consul.consul.fsm.persist + name: consul_fsm_persist + labels: {} + - match: consul.raft.fsm.apply + name: consul_raft_fsm_apply + labels: {} + - match: consul.raft.leader.lastContact + name: consul_raft_leader_lastcontact + labels: {} + - match: consul.raft.leader.dispatchLog + name: consul_raft_leader_dispatchLog + labels: {} + - match: consul.raft.commitTime + name: consul_raft_commitTime + labels: {} + - match: consul.raft.replication.appendEntries.logs.*.*.*.* + name: consul_raft_replication_appendEntries_logs + labels: + query: ${1}.${2}.${3}.${4} + - match: consul.raft.replication.appendEntries.rpc.*.*.*.* + name: consul_raft_replication_appendEntries_rpc + labels: + query: ${1}.${2}.${3}.${4} + - match: consul.raft.replication.heartbeat.*.*.*.* + name: consul_raft_replication_heartbeat + labels: + query: ${1}.${2}.${3}.${4} + - match: consul.consul.rpc.request + name: consul_rpc_requests + labels: {} + - match: consul.consul.rpc.accept_conn + name: consul_rpc_accept_conn + labels: {} + - match: consul.memberlist.udp.* + name: consul_memberlist_udp + labels: + type: $1 + - match: consul.memberlist.tcp.* + name: consul_memberlist_tcp + labels: + type: $1 + - match: consul.memberlist.gossip + name: consul_memberlist_gossip + labels: {} + - match: consul.memberlist.probeNode + name: consul_memberlist_probenode + labels: {} + - match: consul.memberlist.pushPullNode + name: consul_memberlist_pushpullnode + labels: {} + - match: consul.http.* + name: consul_http_request + labels: + method: $1 + path: / + - match: consul.http.*.* + name: consul_http_request + labels: + method: $1 + path: /$2 + - match: consul.http.*.*.* + name: consul_http_request + labels: + method: $1 + path: /$2/$3 + - match: consul.http.*.*.*.* + name: consul_http_request + labels: + method: $1 + path: /$2/$3/$4 + - match: consul.http.*.*.*.*.* + name: consul_http_request + labels: + method: $1 + path: /$2/$3/$4/$5 + - match: consul.consul.leader.barrier + name: consul_leader_barrier + labels: {} + - match: consul.consul.leader.reconcileMember + name: consul_leader_reconcileMember + labels: {} + - match: consul.consul.leader.reconcile + name: consul_leader_reconcile + labels: {} + - match: consul.consul.fsm.coordinate.batch-update + name: consul_fsm_coordinate_batch_update + labels: {} + - match: consul.consul.fsm.autopilot + name: consul_fsm_autopilot + labels: {} + - match: consul.consul.fsm.kvs.cas + name: consul_fsm_kvs_cas + labels: {} + - match: consul.consul.fsm.register + name: consul_fsm_register + labels: {} + - match: consul.consul.fsm.deregister + name: consul_fsm_deregister + labels: {} + - match: consul.consul.fsm.tombstone.reap + name: consul_fsm_tombstone_reap + labels: {} + - match: consul.consul.catalog.register + name: consul_catalog_register + labels: {} + - match: consul.consul.catalog.deregister + name: consul_catalog_deregister + labels: {} + - match: consul.consul.leader.reapTombstones + name: consul_leader_reapTombstones + labels: {} diff --git a/production/helm/loki-distributed/templates/consul/deployment.yaml b/production/helm/loki-distributed/templates/consul/deployment.yaml new file mode 100644 index 000000000000..0e605306bc03 --- /dev/null +++ b/production/helm/loki-distributed/templates/consul/deployment.yaml @@ -0,0 +1,127 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "loki.fullname" . }}-consul + labels: + name: consul + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + {{- toYaml .Values.consul.annotations | nindent 4 }} +spec: + minReadySeconds: 10 + replicas: {{ .Values.consul.replicas }} + selector: + matchLabels: + name: consul + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} + strategy: + {{- toYaml .Values.consul.updateStrategy | nindent 4 }} + template: + metadata: + labels: + name: consul + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/consul/configmap.yaml") . | sha256sum }} + {{- with .Values.consul.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + affinity: + {{- toYaml .Values.consul.affinity | nindent 8 }} + nodeSelector: + {{- toYaml .Values.consul.nodeSelector | nindent 8 }} + tolerations: + {{- toYaml .Values.consul.tolerations | nindent 8 }} + containers: + - name: consul + args: + - agent + - -ui + - -server + - -client=0.0.0.0 + - -config-file=/etc/config/consul-config.json + - -bootstrap-expect=1 + env: + - name: CHECKPOINT_DISABLE + value: "1" + image: "{{ .Values.consul.image.repository }}:{{ .Values.consul.image.tag }}" + imagePullPolicy: {{ .Values.consul.image.pullPolicy }} + ports: + - containerPort: 8300 + name: server + protocol: TCP + - containerPort: 8301 + name: serf + protocol: TCP + - containerPort: 8400 + name: client + protocol: TCP + - containerPort: 8500 + name: api + protocol: TCP + resources: + {{- toYaml .Values.consul.resources | nindent 10 }} + volumeMounts: + - mountPath: /etc/config + name: consul + - args: + - --namespace=$(POD_NAMESPACE) + - --pod-name=$(POD_NAME) + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: weaveworks/consul-sidekick:master-f18ad13 + imagePullPolicy: IfNotPresent + name: sidekick + resources: {} + volumeMounts: + - mountPath: /etc/config + name: consul + - args: + - --web.listen-address=:8000 + - --statsd.mapping-config=/etc/config/mapping + image: prom/statsd-exporter:v0.8.1 + imagePullPolicy: IfNotPresent + name: statsd-exporter + ports: + - containerPort: 8000 + name: http-metrics + protocol: TCP + resources: {} + volumeMounts: + - mountPath: /etc/config + name: consul + - args: + - --consul.server=localhost:8500 + - --web.listen-address=:9107 + - --consul.timeout=1s + image: prom/consul-exporter:v0.4.0 + imagePullPolicy: IfNotPresent + name: consul-exporter + ports: + - containerPort: 9107 + name: http-metrics + protocol: TCP + resources: {} + volumeMounts: + - mountPath: /etc/config + name: consul + volumes: + - configMap: + defaultMode: 420 + name: {{ template "loki.fullname" . }}-consul-config + name: consul \ No newline at end of file diff --git a/production/helm/loki-distributed/templates/consul/service.yaml b/production/helm/loki-distributed/templates/consul/service.yaml new file mode 100644 index 000000000000..b24ba0081954 --- /dev/null +++ b/production/helm/loki-distributed/templates/consul/service.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "loki.fullname" . }}-consul-svc + labels: + name: consul-svc + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + type: ClusterIP + ports: + - name: consul-server + port: 8300 + protocol: TCP + targetPort: 8300 + - name: consul-serf + port: 8301 + protocol: TCP + targetPort: 8301 + - name: consul-client + port: 8400 + protocol: TCP + targetPort: 8400 + - name: consul-api + port: 8500 + protocol: TCP + targetPort: 8500 + - name: statsd-exporter-http-metrics + port: 8000 + protocol: TCP + targetPort: 8000 + - name: consul-exporter-http-metrics + port: 9107 + protocol: TCP + targetPort: 9107 + selector: + name: consul + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} \ No newline at end of file diff --git a/production/helm/loki-distributed/values.yaml b/production/helm/loki-distributed/values.yaml new file mode 100644 index 000000000000..7d78d858cd9d --- /dev/null +++ b/production/helm/loki-distributed/values.yaml @@ -0,0 +1,176 @@ + +consul: + annotations: {} + replicas: 1 + resources: + requests: + cpu: 100m + memory: 500Mi + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + nodeSelector: {} + tolerations: [] + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: consul + topologyKey: kubernetes.io/hostname + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "http-metrics" + image: + repository: consul + tag: 1.4.0 + pullPolicy: IfNotPresent + +config: + auth_enabled: false + ingester: + chunk_idle_period: 15m + chunk_block_size: 262144 + lifecycler: + ring: + kvstore: + store: inmemory + replication_factor: 1 + + ## Different ring configs can be used. E.g. Consul + # ring: + # store: consul + # replication_factor: 1 + # consul: + # host: "consul:8500" + # prefix: "" + # httpclienttimeout: "20s" + # consistentreads: true + limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + schema_config: + configs: + - from: 2018-04-15 + store: boltdb + object_store: filesystem + schema: v9 + index: + prefix: index_ + period: 168h + server: + http_listen_port: 3100 + storage_config: + boltdb: + directory: /data/loki/index + filesystem: + directory: /data/loki/chunks + chunk_store_config: + max_look_back_period: 0 + table_manager: + retention_deletes_enabled: false + retention_period: 0 + +image: + repository: grafana/loki + tag: v0.1.0 + pullPolicy: IfNotPresent + +## Additional Loki container arguments, e.g. log level (debug, info, warn, error) +extraArgs: {} + # log.level: debug + +livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + +## Enable persistence using Persistent Volume Claims +networkPolicy: + enabled: false + +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## If you set enabled as "True", you need : +## - create a pv which above 10Gi and has same namespace with loki +## - keep storageClassName same with below setting +persistence: + enabled: false + accessModes: + - ReadWriteOnce + size: 10Gi + storageClassName: default + annotations: {} + # subPath: "" + # existingClaim: + +## Pod Labels +podLabels: {} + +## Pod Annotations +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "http-metrics" + +podManagementPolicy: OrderedReady + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +rbac: + create: true + pspEnabled: true + +readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + +replicas: 1 + +resources: {} +# limits: +# cpu: 200m +# memory: 256Mi +# requests: +# cpu: 100m +# memory: 128Mi + +securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + +service: + type: ClusterIP + nodePort: + port: 3100 + annotations: {} + labels: {} + +serviceAccount: + create: true + name: + +terminationGracePeriodSeconds: 30 + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +# The values to set in the PodDisruptionBudget spec +# If not set then a PodDisruptionBudget will not be created +podDisruptionBudget: {} +# minAvailable: 1 +# maxUnavailable: 1 + +updateStrategy: + type: RollingUpdate From 421bd3ba993635a5ed01ca4604df49ab8e371339 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 19 Jun 2019 16:11:04 -0400 Subject: [PATCH 02/25] ingester are now querying storage --- pkg/chunkenc/gzip.go | 13 +- pkg/ingester/flush_test.go | 9 + pkg/ingester/ingester.go | 28 ++- pkg/ingester/ingester_test.go | 8 + pkg/ingester/instance.go | 13 +- pkg/loki/loki.go | 3 +- pkg/loki/modules.go | 3 +- pkg/querier/querier.go | 33 ++-- pkg/{querier => storage}/store.go | 49 ++++- production/helm/loki-distributed/.helmignore | 22 --- production/helm/loki-distributed/Chart.yaml | 13 -- .../loki-distributed/templates/_helpers.tpl | 32 ---- production/helm/loki-distributed/values.yaml | 176 ------------------ production/helm/loki/templates/_helpers.tpl | 20 ++ .../templates/consul/configmap.yaml | 2 + .../templates/consul/deployment.yaml | 4 +- .../templates/consul/service.yaml | 4 +- .../helm/loki/templates/networkpolicy.yaml | 2 + production/helm/loki/templates/pdb.yaml | 1 + production/helm/loki/templates/secret.yaml | 2 +- .../helm/loki/templates/service-headless.yaml | 1 + production/helm/loki/templates/service.yaml | 1 + .../helm/loki/templates/statefulset.yaml | 3 + production/helm/loki/values.yaml | 51 +++-- 24 files changed, 207 insertions(+), 286 deletions(-) rename pkg/{querier => storage}/store.go (85%) delete mode 100644 production/helm/loki-distributed/.helmignore delete mode 100644 production/helm/loki-distributed/Chart.yaml delete mode 100644 production/helm/loki-distributed/templates/_helpers.tpl delete mode 100644 production/helm/loki-distributed/values.yaml rename production/helm/{loki-distributed => loki}/templates/consul/configmap.yaml (99%) rename production/helm/{loki-distributed => loki}/templates/consul/deployment.yaml (98%) rename production/helm/{loki-distributed => loki}/templates/consul/service.yaml (90%) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 634728a6b7de..733df813578d 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -9,6 +9,7 @@ import ( "hash/crc32" "io" "math" + "sync" "time" "github.com/grafana/loki/pkg/logproto" @@ -96,8 +97,18 @@ func (hb *headBlock) append(ts int64, line string) error { return nil } +var bufferPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, +} + func (hb *headBlock) serialise(cw func(w io.Writer) CompressionWriter) ([]byte, error) { - buf := &bytes.Buffer{} + buf := bufferPool.Get().(*bytes.Buffer) + defer func() { + buf.Reset() + bufferPool.Put(buf) + }() encBuf := make([]byte, binary.MaxVarintLen64) compressedWriter := cw(buf) for _, logEntry := range hb.entries { diff --git a/pkg/ingester/flush_test.go b/pkg/ingester/flush_test.go index 601134807c06..4b2f850db208 100644 --- a/pkg/ingester/flush_test.go +++ b/pkg/ingester/flush_test.go @@ -11,6 +11,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/grafana/loki/pkg/chunkenc" + "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" @@ -102,6 +103,14 @@ func (s *testStore) Put(ctx context.Context, chunks []chunk.Chunk) error { return nil } +func (s *testStore) IsLocal() bool { + return false +} + +func (s *testStore) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) { + return nil, nil +} + func (s *testStore) Stop() {} func pushTestSamples(t *testing.T, ing logproto.PusherServer) ([]string, map[string][]*logproto.Stream) { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 05638f37174d..07522e6a316e 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -17,6 +17,8 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/util" + "github.com/grafana/loki/pkg/helpers" + "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" ) @@ -74,6 +76,8 @@ type Ingester struct { // ChunkStore is the interface we need to store chunks. type ChunkStore interface { Put(ctx context.Context, chunks []chunk.Chunk) error + IsLocal() bool + LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) } // New makes a new Ingester. @@ -183,8 +187,30 @@ func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querie return err } + iters := []iter.EntryIterator{} instance := i.getOrCreateInstance(instanceID) - return instance.Query(req, queryServer) + instanceIter, err := instance.Query(req) + if err != nil { + return err + } + iters = append(iters, instanceIter) + + // we should also query the store if: + // - it is local to the ingester + // - the request is beyond what the ingester has in memory + if i.store.IsLocal() && req.End.After(instance.createdAt.Add(i.cfg.RetainPeriod)) { + req.Start = instance.createdAt.Add(i.cfg.RetainPeriod) + lazyIter, err := i.store.LazyQuery(queryServer.Context(), req) + if err != nil { + return err + } + iters = append(iters, lazyIter) + } + + iter := iter.NewHeapIterator(iters, req.Direction) + defer helpers.LogError("closing iterator", iter.Close) + + return sendBatches(iter, queryServer, req.Limit) } // Label returns the set of labels for the stream this ingester knows about. diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 16f395b5becb..5a488d6455f4 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" "github.com/stretchr/testify/require" "github.com/weaveworks/common/user" @@ -110,6 +111,13 @@ func (s *mockStore) Put(ctx context.Context, chunks []chunk.Chunk) error { return nil } +func (s *mockStore) IsLocal() bool { + return false +} +func (s *mockStore) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) { + return nil, nil +} + type mockQuerierServer struct { ctx context.Context resps []*logproto.QueryResponse diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 4de1930f6cd8..3d79fd6bb92f 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -3,6 +3,7 @@ package ingester import ( "context" "sync" + "time" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -55,6 +56,7 @@ type instance struct { blockSize int tailers map[uint32]*tailer tailerMtx sync.RWMutex + createdAt time.Time } func newInstance(instanceID string, blockSize int) *instance { @@ -68,6 +70,7 @@ func newInstance(instanceID string, blockSize int) *instance { blockSize: blockSize, tailers: map[uint32]*tailer{}, + createdAt: time.Now(), } } @@ -102,10 +105,10 @@ func (i *instance) Push(ctx context.Context, req *logproto.PushRequest) error { return appendErr } -func (i *instance) Query(req *logproto.QueryRequest, queryServer logproto.Querier_QueryServer) error { +func (i *instance) Query(req *logproto.QueryRequest) (iter.EntryIterator, error) { expr, err := logql.ParseExpr(req.Query) if err != nil { - return err + return nil, err } if req.Regex != "" { @@ -123,11 +126,9 @@ func (i *instance) Query(req *logproto.QueryRequest, queryServer logproto.Querie iter, err := expr.Eval(querier) if err != nil { - return err + return nil, err } - defer helpers.LogError("closing iterator", iter.Close) - - return sendBatches(iter, queryServer, req.Limit) + return iter, nil } func (i *instance) Label(_ context.Context, req *logproto.LabelRequest) (*logproto.LabelResponse, error) { diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 2d1d143a4b7a..0622749fdd59 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -20,6 +20,7 @@ import ( "github.com/grafana/loki/pkg/ingester" "github.com/grafana/loki/pkg/ingester/client" "github.com/grafana/loki/pkg/querier" + loki_storage "github.com/grafana/loki/pkg/storage" ) // Config is the root config for Loki. @@ -69,7 +70,7 @@ type Loki struct { distributor *distributor.Distributor ingester *ingester.Ingester querier *querier.Querier - store chunk.Store + store loki_storage.Store tableManager *chunk.TableManager httpAuthMiddleware middleware.Interface diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 9a710fa742a1..3c871088083d 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/loki/pkg/ingester" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/querier" + loki_storage "github.com/grafana/loki/pkg/storage" ) const maxChunkAgeForTableManager = 12 * time.Hour @@ -218,7 +219,7 @@ func (t *Loki) stopTableManager() error { } func (t *Loki) initStore() (err error) { - t.store, err = storage.NewStore(t.cfg.StorageConfig, t.cfg.ChunkStoreConfig, t.cfg.SchemaConfig, t.overrides) + t.store, err = loki_storage.NewStore(t.cfg.StorageConfig, t.cfg.ChunkStoreConfig, t.cfg.SchemaConfig, t.overrides) return } diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 5a2a5e2ff18b..3e13a2f09710 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -5,7 +5,6 @@ import ( "flag" "time" - "github.com/cortexproject/cortex/pkg/chunk" cortex_client "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/util" @@ -17,6 +16,7 @@ import ( "github.com/grafana/loki/pkg/ingester/client" "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/storage" ) // Config for a querier. @@ -32,11 +32,11 @@ type Querier struct { cfg Config ring ring.ReadRing pool *cortex_client.Pool - store chunk.Store + store storage.Store } // New makes a new Querier. -func New(cfg Config, clientCfg client.Config, ring ring.ReadRing, store chunk.Store) (*Querier, error) { +func New(cfg Config, clientCfg client.Config, ring ring.ReadRing, store storage.Store) (*Querier, error) { factory := func(addr string) (grpc_health_v1.HealthClient, error) { return client.New(clientCfg, addr) } @@ -106,17 +106,23 @@ func (q *Querier) forGivenIngesters(replicationSet ring.ReplicationSet, f func(l // Query does the heavy lifting for an actual query. func (q *Querier) Query(ctx context.Context, req *logproto.QueryRequest) (*logproto.QueryResponse, error) { + var iterators []iter.EntryIterator + ingesterIterators, err := q.queryIngesters(ctx, req) if err != nil { return nil, err } + iterators = append(iterators, ingesterIterators...) - chunkStoreIterators, err := q.queryStore(ctx, req) - if err != nil { - return nil, err + // if the store is local to ingester, we want to query it from the ingester not from the querier + if !q.store.IsLocal() { + chunkStoreIterators, err := q.store.LazyQuery(ctx, req) + if err != nil { + return nil, err + } + iterators = append(iterators, chunkStoreIterators) } - iterators := append(ingesterIterators, chunkStoreIterators) iterator := iter.NewHeapIterator(iterators, req.Direction) defer helpers.LogError("closing iterator", iterator.Close) @@ -277,17 +283,22 @@ func (q *Querier) queryDroppedStreams(ctx context.Context, req *logproto.TailReq return nil, err } + var iterators []iter.EntryIterator ingesterIterators := make([]iter.EntryIterator, len(clients)) for i := range clients { ingesterIterators[i] = iter.NewQueryClientIterator(clients[i].response.(logproto.Querier_QueryClient), query.Direction) } + iterators = append(iterators, ingesterIterators...) - chunkStoreIterators, err := q.queryStore(ctx, &query) - if err != nil { - return nil, err + // if the store is local to ingester, we want to query it from the ingester not from the querier + if !q.store.IsLocal() { + chunkStoreIterators, err := q.store.LazyQuery(ctx, &query) + if err != nil { + return nil, err + } + iterators = append(iterators, chunkStoreIterators) } - iterators := append(ingesterIterators, chunkStoreIterators) return iter.NewHeapIterator(iterators, query.Direction), nil } diff --git a/pkg/querier/store.go b/pkg/storage/store.go similarity index 85% rename from pkg/querier/store.go rename to pkg/storage/store.go index 9a2c90219a9b..8847a891706f 100644 --- a/pkg/querier/store.go +++ b/pkg/storage/store.go @@ -1,21 +1,56 @@ -package querier +package storage import ( "context" "sort" "github.com/cortexproject/cortex/pkg/chunk" - "github.com/opentracing/opentracing-go" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/labels" - + "github.com/cortexproject/cortex/pkg/chunk/storage" + "github.com/cortexproject/cortex/pkg/util/validation" "github.com/grafana/loki/pkg/chunkenc" "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" ) -func (q Querier) queryStore(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) { +type Store interface { + chunk.Store + IsLocal() bool + LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) +} + +type store struct { + chunk.Store + isLocal bool +} + +func NewStore(cfg storage.Config, storeCfg chunk.StoreConfig, schemaCfg chunk.SchemaConfig, limits *validation.Overrides) (Store, error) { + s, err := storage.NewStore(cfg, storeCfg, schemaCfg, limits) + if err != nil { + return nil, err + } + var isLocal bool + for _, cfg := range schemaCfg.Configs { + if cfg.ObjectType == "filesystem" || cfg.IndexType == "boltdb" { + isLocal = true + break + } + } + return &store{ + Store: s, + isLocal: isLocal, + }, nil +} + +// IsLocal tells if the storage for chunks and indexes is local. +func (s *store) IsLocal() bool { + return s.isLocal +} + +func (s *store) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) { expr, err := logql.ParseExpr(req.Query) if err != nil { return nil, err @@ -33,7 +68,7 @@ func (q Querier) queryStore(ctx context.Context, req *logproto.QueryRequest) (it matchers = append(matchers, nameLabelMatcher) from, through := model.TimeFromUnixNano(req.Start.UnixNano()), model.TimeFromUnixNano(req.End.UnixNano()) - chks, fetchers, err := q.store.GetChunkRefs(ctx, from, through, matchers...) + chks, fetchers, err := s.GetChunkRefs(ctx, from, through, matchers...) if err != nil { return nil, err } diff --git a/production/helm/loki-distributed/.helmignore b/production/helm/loki-distributed/.helmignore deleted file mode 100644 index 50af03172541..000000000000 --- a/production/helm/loki-distributed/.helmignore +++ /dev/null @@ -1,22 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/production/helm/loki-distributed/Chart.yaml b/production/helm/loki-distributed/Chart.yaml deleted file mode 100644 index 8b003d090d11..000000000000 --- a/production/helm/loki-distributed/Chart.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: loki-distributed -version: 0.1.0 -appVersion: 0.0.1 -kubeVersion: "^1.10.0-0" -description: "Loki: like Prometheus, but for logs." -home: https://grafana.com/loki -icon: https://github.com/grafana/loki/raw/master/docs/logo.png -sources: - - https://github.com/grafana/loki -maintainers: - - name: Loki Maintainers - email: lokiproject@googlegroups.com -engine: gotpl diff --git a/production/helm/loki-distributed/templates/_helpers.tpl b/production/helm/loki-distributed/templates/_helpers.tpl deleted file mode 100644 index 90d9de08e2b5..000000000000 --- a/production/helm/loki-distributed/templates/_helpers.tpl +++ /dev/null @@ -1,32 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "loki.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "loki.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "loki.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/production/helm/loki-distributed/values.yaml b/production/helm/loki-distributed/values.yaml deleted file mode 100644 index 7d78d858cd9d..000000000000 --- a/production/helm/loki-distributed/values.yaml +++ /dev/null @@ -1,176 +0,0 @@ - -consul: - annotations: {} - replicas: 1 - resources: - requests: - cpu: 100m - memory: 500Mi - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - nodeSelector: {} - tolerations: [] - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - name: consul - topologyKey: kubernetes.io/hostname - podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "http-metrics" - image: - repository: consul - tag: 1.4.0 - pullPolicy: IfNotPresent - -config: - auth_enabled: false - ingester: - chunk_idle_period: 15m - chunk_block_size: 262144 - lifecycler: - ring: - kvstore: - store: inmemory - replication_factor: 1 - - ## Different ring configs can be used. E.g. Consul - # ring: - # store: consul - # replication_factor: 1 - # consul: - # host: "consul:8500" - # prefix: "" - # httpclienttimeout: "20s" - # consistentreads: true - limits_config: - enforce_metric_name: false - reject_old_samples: true - reject_old_samples_max_age: 168h - schema_config: - configs: - - from: 2018-04-15 - store: boltdb - object_store: filesystem - schema: v9 - index: - prefix: index_ - period: 168h - server: - http_listen_port: 3100 - storage_config: - boltdb: - directory: /data/loki/index - filesystem: - directory: /data/loki/chunks - chunk_store_config: - max_look_back_period: 0 - table_manager: - retention_deletes_enabled: false - retention_period: 0 - -image: - repository: grafana/loki - tag: v0.1.0 - pullPolicy: IfNotPresent - -## Additional Loki container arguments, e.g. log level (debug, info, warn, error) -extraArgs: {} - # log.level: debug - -livenessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - -## Enable persistence using Persistent Volume Claims -networkPolicy: - enabled: false - -## ref: https://kubernetes.io/docs/user-guide/node-selection/ -nodeSelector: {} - -## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ -## If you set enabled as "True", you need : -## - create a pv which above 10Gi and has same namespace with loki -## - keep storageClassName same with below setting -persistence: - enabled: false - accessModes: - - ReadWriteOnce - size: 10Gi - storageClassName: default - annotations: {} - # subPath: "" - # existingClaim: - -## Pod Labels -podLabels: {} - -## Pod Annotations -podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "http-metrics" - -podManagementPolicy: OrderedReady - -## Assign a PriorityClassName to pods if set -# priorityClassName: - -rbac: - create: true - pspEnabled: true - -readinessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - -replicas: 1 - -resources: {} -# limits: -# cpu: 200m -# memory: 256Mi -# requests: -# cpu: 100m -# memory: 128Mi - -securityContext: - fsGroup: 10001 - runAsGroup: 10001 - runAsNonRoot: true - runAsUser: 10001 - -service: - type: ClusterIP - nodePort: - port: 3100 - annotations: {} - labels: {} - -serviceAccount: - create: true - name: - -terminationGracePeriodSeconds: 30 - -## Tolerations for pod assignment -## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -tolerations: [] - -# The values to set in the PodDisruptionBudget spec -# If not set then a PodDisruptionBudget will not be created -podDisruptionBudget: {} -# minAvailable: 1 -# maxUnavailable: 1 - -updateStrategy: - type: RollingUpdate diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl index 2e333aae60f8..02b5713715da 100644 --- a/production/helm/loki/templates/_helpers.tpl +++ b/production/helm/loki/templates/_helpers.tpl @@ -41,3 +41,23 @@ Create the name of the service account {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Create a config to merge with the current loki config to overrides values +*/}} +{{- define "loki.config.overrides" -}} +ingester: + lifecycler: + ring: + kvstore: + {{ if (include "loki.deployConsul" .)}} + consul: + host: {{ printf "%s-consul-svc.%s.svc.cluster.local:8500" (include "loki.fullname" .) .Release.Namespace}} + {{end}} +{{- end -}} + +{{- define "loki.deployConsul" -}} + {{- if and (eq .Values.config.ingester.lifecycler.ring.kvstore.store "consul") .Values.consul.enabled -}} + {{true}} + {{- end -}} +{{- end -}} diff --git a/production/helm/loki-distributed/templates/consul/configmap.yaml b/production/helm/loki/templates/consul/configmap.yaml similarity index 99% rename from production/helm/loki-distributed/templates/consul/configmap.yaml rename to production/helm/loki/templates/consul/configmap.yaml index dcd3800fc34a..70a4d6935d24 100644 --- a/production/helm/loki-distributed/templates/consul/configmap.yaml +++ b/production/helm/loki/templates/consul/configmap.yaml @@ -1,3 +1,4 @@ + {{- if (include "loki.deployConsul" .) }} apiVersion: v1 kind: ConfigMap metadata: @@ -215,3 +216,4 @@ data: - match: consul.consul.leader.reapTombstones name: consul_leader_reapTombstones labels: {} +{{- end }} \ No newline at end of file diff --git a/production/helm/loki-distributed/templates/consul/deployment.yaml b/production/helm/loki/templates/consul/deployment.yaml similarity index 98% rename from production/helm/loki-distributed/templates/consul/deployment.yaml rename to production/helm/loki/templates/consul/deployment.yaml index 0e605306bc03..799022db5ae1 100644 --- a/production/helm/loki-distributed/templates/consul/deployment.yaml +++ b/production/helm/loki/templates/consul/deployment.yaml @@ -1,3 +1,4 @@ + {{- if (include "loki.deployConsul" .) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -124,4 +125,5 @@ spec: - configMap: defaultMode: 420 name: {{ template "loki.fullname" . }}-consul-config - name: consul \ No newline at end of file + name: consul +{{- end }} \ No newline at end of file diff --git a/production/helm/loki-distributed/templates/consul/service.yaml b/production/helm/loki/templates/consul/service.yaml similarity index 90% rename from production/helm/loki-distributed/templates/consul/service.yaml rename to production/helm/loki/templates/consul/service.yaml index b24ba0081954..bd175edba39f 100644 --- a/production/helm/loki-distributed/templates/consul/service.yaml +++ b/production/helm/loki/templates/consul/service.yaml @@ -1,3 +1,4 @@ + {{- if (include "loki.deployConsul" .) }} apiVersion: v1 kind: Service metadata: @@ -38,4 +39,5 @@ spec: selector: name: consul app: {{ template "loki.name" . }} - release: {{ .Release.Name }} \ No newline at end of file + release: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/production/helm/loki/templates/networkpolicy.yaml b/production/helm/loki/templates/networkpolicy.yaml index 5becbcdb5915..0e71a9bf09aa 100644 --- a/production/helm/loki/templates/networkpolicy.yaml +++ b/production/helm/loki/templates/networkpolicy.yaml @@ -11,6 +11,7 @@ metadata: spec: podSelector: matchLabels: + name: loki name: {{ template "loki.fullname" . }} app: {{ template "loki.name" . }} release: {{ .Release.Name }} @@ -18,6 +19,7 @@ spec: - from: - podSelector: matchLabels: + name: loki app: {{ template "promtail.name" . }} release: {{ .Release.Name }} - ports: diff --git a/production/helm/loki/templates/pdb.yaml b/production/helm/loki/templates/pdb.yaml index 27093f5d8ed6..6c3bb54c8c87 100644 --- a/production/helm/loki/templates/pdb.yaml +++ b/production/helm/loki/templates/pdb.yaml @@ -11,6 +11,7 @@ metadata: spec: selector: matchLabels: + name: loki app: {{ template "loki.name" . }} {{ toYaml .Values.podDisruptionBudget | indent 2 }} {{- end -}} diff --git a/production/helm/loki/templates/secret.yaml b/production/helm/loki/templates/secret.yaml index 95b16fad4be9..f52f7a3c6376 100644 --- a/production/helm/loki/templates/secret.yaml +++ b/production/helm/loki/templates/secret.yaml @@ -8,4 +8,4 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - loki.yaml: {{ tpl (toYaml .Values.config) . | b64enc}} + loki.yaml: {{ tpl (toYaml (mergeOverwrite .Values.config (fromYaml (include "loki.config.overrides" . )))) . | b64enc}} diff --git a/production/helm/loki/templates/service-headless.yaml b/production/helm/loki/templates/service-headless.yaml index dbc127b3bea3..3167d861b6a6 100644 --- a/production/helm/loki/templates/service-headless.yaml +++ b/production/helm/loki/templates/service-headless.yaml @@ -15,5 +15,6 @@ spec: name: http-metrics targetPort: http-metrics selector: + name: loki app: {{ template "loki.name" . }} release: {{ .Release.Name }} diff --git a/production/helm/loki/templates/service.yaml b/production/helm/loki/templates/service.yaml index 9a8c1da58592..ebbb11adb195 100644 --- a/production/helm/loki/templates/service.yaml +++ b/production/helm/loki/templates/service.yaml @@ -26,5 +26,6 @@ spec: nodePort: {{ .Values.service.nodePort }} {{- end }} selector: + name: loki app: {{ template "loki.name" . }} release: {{ .Release.Name }} diff --git a/production/helm/loki/templates/statefulset.yaml b/production/helm/loki/templates/statefulset.yaml index a1813b130aa2..061f67569e5b 100644 --- a/production/helm/loki/templates/statefulset.yaml +++ b/production/helm/loki/templates/statefulset.yaml @@ -3,6 +3,7 @@ kind: StatefulSet metadata: name: {{ template "loki.fullname" . }} labels: + name: loki app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} release: {{ .Release.Name }} @@ -14,6 +15,7 @@ spec: replicas: {{ .Values.replicas }} selector: matchLabels: + name: loki app: {{ template "loki.name" . }} release: {{ .Release.Name }} serviceName: {{ template "loki.fullname" . }}-headless @@ -22,6 +24,7 @@ spec: template: metadata: labels: + name: loki app: {{ template "loki.name" . }} name: {{ template "loki.name" . }} release: {{ .Release.Name }} diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index c6725a5f8b9b..075d29700503 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -26,18 +26,12 @@ config: lifecycler: ring: kvstore: - store: inmemory + store: consul + consul: + consistentreads: true + httpclienttimeout: 20s + prefix: "" replication_factor: 1 - - ## Different ring configs can be used. E.g. Consul - # ring: - # store: consul - # replication_factor: 1 - # consul: - # host: "consul:8500" - # prefix: "" - # httpclienttimeout: "20s" - # consistentreads: true limits_config: enforce_metric_name: false reject_old_samples: true @@ -123,7 +117,7 @@ readinessProbe: port: http-metrics initialDelaySeconds: 45 -replicas: 1 +replicas: 3 resources: {} # limits: @@ -164,3 +158,36 @@ podDisruptionBudget: {} updateStrategy: type: RollingUpdate + +# consul configuration used by the ring if consul is selected +consul: + # deploy consul if the kvstore is consul, set to false to use your own consul instance. + # The host can be set by using config.ingester.lifecycler.ring.kvstore.consul.host + enabled: true + annotations: {} + replicas: 1 + resources: + requests: + cpu: 100m + memory: 500Mi + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + nodeSelector: {} + tolerations: [] + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: consul + topologyKey: kubernetes.io/hostname + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "http-metrics" + image: + repository: consul + tag: 1.4.0 + pullPolicy: IfNotPresent From e717e7b033e60d5ea59702befa5e109d31fd743f Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 20 Jun 2019 13:01:37 -0400 Subject: [PATCH 03/25] tweak chart default and fix ingester store query --- pkg/chunkenc/gzip.go | 13 +------------ pkg/ingester/ingester.go | 14 ++++++++++++-- production/helm/loki/Chart.yaml | 2 +- production/helm/loki/values.yaml | 4 ++-- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 733df813578d..634728a6b7de 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -9,7 +9,6 @@ import ( "hash/crc32" "io" "math" - "sync" "time" "github.com/grafana/loki/pkg/logproto" @@ -97,18 +96,8 @@ func (hb *headBlock) append(ts int64, line string) error { return nil } -var bufferPool = sync.Pool{ - New: func() interface{} { - return new(bytes.Buffer) - }, -} - func (hb *headBlock) serialise(cw func(w io.Writer) CompressionWriter) ([]byte, error) { - buf := bufferPool.Get().(*bytes.Buffer) - defer func() { - buf.Reset() - bufferPool.Put(buf) - }() + buf := &bytes.Buffer{} encBuf := make([]byte, binary.MaxVarintLen64) compressedWriter := cw(buf) for _, logEntry := range hb.entries { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 07522e6a316e..9436b5afb80e 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -195,11 +195,21 @@ func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querie } iters = append(iters, instanceIter) + // last sample in memory is either now minus retain period + // or since the instance was created. + lastInmemory := time.Now().Add(-i.cfg.RetainPeriod) + if instance.createdAt.After(lastInmemory) { + lastInmemory = instance.createdAt + } + // we should also query the store if: // - it is local to the ingester // - the request is beyond what the ingester has in memory - if i.store.IsLocal() && req.End.After(instance.createdAt.Add(i.cfg.RetainPeriod)) { - req.Start = instance.createdAt.Add(i.cfg.RetainPeriod) + if i.store.IsLocal() && req.Start.Before(lastInmemory) { + // we query the remaining data from the store. + if req.End.After(lastInmemory) { + req.End = lastInmemory + } lazyIter, err := i.store.LazyQuery(queryServer.Context(), req) if err != nil { return err diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 8703a5fff724..4be7c928c860 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.10.0 +version: 0.11.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 075d29700503..80f4ce43526c 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -71,7 +71,7 @@ livenessProbe: httpGet: path: /ready port: http-metrics - initialDelaySeconds: 45 + initialDelaySeconds: 90 ## Enable persistence using Persistent Volume Claims networkPolicy: @@ -115,7 +115,7 @@ readinessProbe: httpGet: path: /ready port: http-metrics - initialDelaySeconds: 45 + initialDelaySeconds: 90 replicas: 3 From eb202ec4b064f8aab84aeb3cf8f3a5b2ca282a9b Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 20 Jun 2019 17:47:07 -0400 Subject: [PATCH 04/25] pool the gzip writer --- pkg/chunkenc/gzip.go | 34 ++++++++++++++---- pkg/chunkenc/gzip_test.go | 73 +++++++++++++++++++++++++++++++++++++++ pkg/chunkenc/interface.go | 5 +++ pkg/querier/querier.go | 3 +- 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 634728a6b7de..4272023a0f34 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -9,6 +9,7 @@ import ( "hash/crc32" "io" "math" + "sync" "time" "github.com/grafana/loki/pkg/logproto" @@ -41,6 +42,26 @@ func newCRC32() hash.Hash32 { return crc32.New(castagnoliTable) } +type compressionPool struct { + sync.Pool +} + +func (p *compressionPool) Get() CompressionWriter { + return p.Pool.Get().(CompressionWriter) +} + +func (p *compressionPool) Put(cw CompressionWriter) { + p.Pool.Put(cw) +} + +var compressionWriters = map[Encoding]CompressionWriterPool{ + EncGZIP: &compressionPool{Pool: sync.Pool{ + New: func() interface{} { + return gzip.NewWriter(nil) + }, + }}, +} + // MemChunk implements compressed log chunks. type MemChunk struct { // The number of uncompressed bytes per block. @@ -53,7 +74,7 @@ type MemChunk struct { head *headBlock encoding Encoding - cw func(w io.Writer) CompressionWriter + cw CompressionWriterPool cr func(r io.Reader) (CompressionReader, error) } @@ -96,10 +117,11 @@ func (hb *headBlock) append(ts int64, line string) error { return nil } -func (hb *headBlock) serialise(cw func(w io.Writer) CompressionWriter) ([]byte, error) { +func (hb *headBlock) serialise(pool CompressionWriterPool) ([]byte, error) { buf := &bytes.Buffer{} encBuf := make([]byte, binary.MaxVarintLen64) - compressedWriter := cw(buf) + compressedWriter := pool.Get() + compressedWriter.Reset(buf) for _, logEntry := range hb.entries { n := binary.PutVarint(encBuf, logEntry.t) _, err := compressedWriter.Write(encBuf[:n]) @@ -117,10 +139,10 @@ func (hb *headBlock) serialise(cw func(w io.Writer) CompressionWriter) ([]byte, return nil, errors.Wrap(err, "appending entry") } } - if err := compressedWriter.Close(); err != nil { + if err := compressedWriter.Flush(); err != nil { return nil, errors.Wrap(err, "flushing pending compress buffer") } - + pool.Put(compressedWriter) return buf.Bytes(), nil } @@ -146,7 +168,7 @@ func NewMemChunkSize(enc Encoding, blockSize int) *MemChunk { switch enc { case EncGZIP: - c.cw = func(w io.Writer) CompressionWriter { return gzip.NewWriter(w) } + c.cw = compressionWriters[EncGZIP] c.cr = func(r io.Reader) (CompressionReader, error) { return gzip.NewReader(r) } default: panic("unknown encoding") diff --git a/pkg/chunkenc/gzip_test.go b/pkg/chunkenc/gzip_test.go index c210889352fa..dc59559906f8 100644 --- a/pkg/chunkenc/gzip_test.go +++ b/pkg/chunkenc/gzip_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "math" + "math/rand" "testing" "time" @@ -215,6 +216,78 @@ func TestGZIPChunkFilling(t *testing.T) { require.Equal(t, int64(lines), i) } +var result []Chunk + +func BenchmarkWriteGZIP(b *testing.B) { + chunks := []Chunk{} + + entry := &logproto.Entry{ + Timestamp: time.Unix(0, 0), + Line: RandString(512), + } + i := int64(0) + + for n := 0; n < b.N; n++ { + c := NewMemChunk(EncGZIP) + // adds until full so we trigger cut which serialize using gzip + for c.SpaceFor(entry) { + c.Append(entry) + entry.Timestamp = time.Unix(0, i) + i++ + } + chunks = append(chunks, c) + } + result = chunks +} + +func BenchmarkReadGZIP(b *testing.B) { + chunks := []Chunk{} + + entry := &logproto.Entry{ + Timestamp: time.Unix(0, 0), + Line: RandString(512), + } + i := int64(0) + + for n := 0; n < 100; n++ { + c := NewMemChunk(EncGZIP) + // adds until full so we trigger cut which serialize using gzip + for c.SpaceFor(entry) { + c.Append(entry) + entry.Timestamp = time.Unix(0, i) + i++ + } + chunks = append(chunks, c) + } + for n := 0; n < b.N; n++ { + for _, c := range chunks { + iter, err := c.Iterator(time.Unix(0, 0), time.Unix(0, 100), logproto.BACKWARD) + if err != nil { + b.Fatal(err) + } + for iter.Next() { + + } + } + } + +} + +const charset = "abcdefghijklmnopqrstuvwxyz" + + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func RandStringWithCharset(length int, charset string) string { + b := make([]byte, length) + for i := range b { + b[i] = charset[rand.Intn(len(charset)-1)] + } + return string(b) +} + +func RandString(length int) string { + return RandStringWithCharset(length, charset) +} + func logprotoEntry(ts int64, line string) *logproto.Entry { return &logproto.Entry{ Timestamp: time.Unix(0, ts), diff --git a/pkg/chunkenc/interface.go b/pkg/chunkenc/interface.go index b01358761ad4..afc1c0683f3a 100644 --- a/pkg/chunkenc/interface.go +++ b/pkg/chunkenc/interface.go @@ -64,3 +64,8 @@ type CompressionReader interface { Read(p []byte) (int, error) Reset(r io.Reader) error } + +type CompressionWriterPool interface { + Get() CompressionWriter + Put(CompressionWriter) +} diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 3e13a2f09710..df3c0a4e01c7 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -114,7 +114,8 @@ func (q *Querier) Query(ctx context.Context, req *logproto.QueryRequest) (*logpr } iterators = append(iterators, ingesterIterators...) - // if the store is local to ingester, we want to query it from the ingester not from the querier + // if the store is local to ingester, we want to query it from the ingester only + // the querier doesn't have access. if !q.store.IsLocal() { chunkStoreIterators, err := q.store.LazyQuery(ctx, req) if err != nil { From 712fd7e8509db5d5179fe84d41ef8458c5680a68 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 26 Jun 2019 20:56:24 -0400 Subject: [PATCH 05/25] add benchmark --- cmd/loki/loki-local-config.yaml | 3 +- pkg/chunkenc/facade.go | 2 +- pkg/chunkenc/gzip.go | 113 +++++++++++++-------------- pkg/chunkenc/gzip_test.go | 25 ++++-- pkg/chunkenc/interface.go | 6 +- pkg/chunkenc/lazy_chunk.go | 28 ++++++- pkg/chunkenc/pool.go | 112 +++++++++++++++++++++++++++ pkg/iter/iterator.go | 78 ++++++++++++------- pkg/storage/hack/main.go | 131 ++++++++++++++++++++++++++++++++ pkg/storage/store_test.go | 82 ++++++++++++++++++++ 10 files changed, 474 insertions(+), 106 deletions(-) create mode 100644 pkg/chunkenc/pool.go create mode 100644 pkg/storage/hack/main.go create mode 100644 pkg/storage/store_test.go diff --git a/cmd/loki/loki-local-config.yaml b/cmd/loki/loki-local-config.yaml index 17e3511305ee..dac5d2841a65 100644 --- a/cmd/loki/loki-local-config.yaml +++ b/cmd/loki/loki-local-config.yaml @@ -10,7 +10,8 @@ ingester: kvstore: store: inmemory replication_factor: 1 - chunk_idle_period: 15m + chunk_idle_period: 5m + chunk_retain_period: 30s schema_config: configs: diff --git a/pkg/chunkenc/facade.go b/pkg/chunkenc/facade.go index e10defe37ece..ef3801d1dc22 100644 --- a/pkg/chunkenc/facade.go +++ b/pkg/chunkenc/facade.go @@ -17,7 +17,7 @@ func init() { }) } -// Facade for compatibility with cortex chunk type, so we can use it's chunk store. +// Facade for compatibility with cortex chunk type, so we can use its chunk store. type Facade struct { c Chunk encoding.Chunk diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 4272023a0f34..39c7a14c6f8a 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -3,13 +3,11 @@ package chunkenc import ( "bufio" "bytes" - "compress/gzip" "encoding/binary" "hash" "hash/crc32" "io" "math" - "sync" "time" "github.com/grafana/loki/pkg/logproto" @@ -42,26 +40,6 @@ func newCRC32() hash.Hash32 { return crc32.New(castagnoliTable) } -type compressionPool struct { - sync.Pool -} - -func (p *compressionPool) Get() CompressionWriter { - return p.Pool.Get().(CompressionWriter) -} - -func (p *compressionPool) Put(cw CompressionWriter) { - p.Pool.Put(cw) -} - -var compressionWriters = map[Encoding]CompressionWriterPool{ - EncGZIP: &compressionPool{Pool: sync.Pool{ - New: func() interface{} { - return gzip.NewWriter(nil) - }, - }}, -} - // MemChunk implements compressed log chunks. type MemChunk struct { // The number of uncompressed bytes per block. @@ -74,8 +52,7 @@ type MemChunk struct { head *headBlock encoding Encoding - cw CompressionWriterPool - cr func(r io.Reader) (CompressionReader, error) + cPool CompressionPool } type block struct { @@ -117,11 +94,10 @@ func (hb *headBlock) append(ts int64, line string) error { return nil } -func (hb *headBlock) serialise(pool CompressionWriterPool) ([]byte, error) { +func (hb *headBlock) serialise(pool CompressionPool) ([]byte, error) { buf := &bytes.Buffer{} encBuf := make([]byte, binary.MaxVarintLen64) - compressedWriter := pool.Get() - compressedWriter.Reset(buf) + compressedWriter := pool.GetWriter(buf) for _, logEntry := range hb.entries { n := binary.PutVarint(encBuf, logEntry.t) _, err := compressedWriter.Write(encBuf[:n]) @@ -139,10 +115,10 @@ func (hb *headBlock) serialise(pool CompressionWriterPool) ([]byte, error) { return nil, errors.Wrap(err, "appending entry") } } - if err := compressedWriter.Flush(); err != nil { + if err := compressedWriter.Close(); err != nil { return nil, errors.Wrap(err, "flushing pending compress buffer") } - pool.Put(compressedWriter) + pool.PutWriter(compressedWriter) return buf.Bytes(), nil } @@ -168,8 +144,7 @@ func NewMemChunkSize(enc Encoding, blockSize int) *MemChunk { switch enc { case EncGZIP: - c.cw = compressionWriters[EncGZIP] - c.cr = func(r io.Reader) (CompressionReader, error) { return gzip.NewReader(r) } + c.cPool = &Gzip default: panic("unknown encoding") } @@ -185,8 +160,8 @@ func NewMemChunk(enc Encoding) *MemChunk { // NewByteChunk returns a MemChunk on the passed bytes. func NewByteChunk(b []byte) (*MemChunk, error) { bc := &MemChunk{ - cr: func(r io.Reader) (CompressionReader, error) { return gzip.NewReader(r) }, - head: &headBlock{}, // Dummy, empty headblock. + cPool: &Gzip, + head: &headBlock{}, // Dummy, empty headblock. } db := decbuf{b: b} @@ -365,7 +340,7 @@ func (c *MemChunk) cut() error { return nil } - b, err := c.head.serialise(c.cw) + b, err := c.head.serialise(c.cPool) if err != nil { return err } @@ -412,7 +387,7 @@ func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction for _, b := range c.blocks { if maxt > b.mint && b.maxt > mint { - it, err := b.iterator(c.cr) + it, err := b.iterator(c.cPool) if err != nil { return nil, err } @@ -436,18 +411,11 @@ func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction return iter.NewEntryIteratorBackward(iterForward) } -func (b block) iterator(cr func(io.Reader) (CompressionReader, error)) (iter.EntryIterator, error) { +func (b block) iterator(pool CompressionPool) (iter.EntryIterator, error) { if len(b.b) == 0 { return emptyIterator, nil } - - r, err := cr(bytes.NewBuffer(b.b)) - if err != nil { - return nil, err - } - - s := bufio.NewReader(r) - return newBufferedIterator(s), nil + return newBufferedIterator(pool, b.b), nil } func (hb *headBlock) iterator(mint, maxt int64) iter.EntryIterator { @@ -499,22 +467,32 @@ func (li *listIterator) Close() error { return nil } func (li *listIterator) Labels() string { return "" } type bufferedIterator struct { - s *bufio.Reader + s *bufio.Reader + reader CompressionReader + pool CompressionPool - curT int64 - curLog string + curT int64 + l uint64 err error buf []byte // The buffer a single entry. decBuf []byte // The buffer for decoding the lengths. + + closed bool } -func newBufferedIterator(s *bufio.Reader) *bufferedIterator { +func newBufferedIterator(pool CompressionPool, b []byte) *bufferedIterator { + buf := bytes.NewBuffer(b) + + r := pool.GetReader(buf) + return &bufferedIterator{ - s: s, - buf: make([]byte, 1024), - decBuf: make([]byte, binary.MaxVarintLen64), + s: BufReaderPool.Get(r), + reader: r, + pool: pool, + buf: LineBufferPool.Get(), + decBuf: IntBinBufferPool.Get(), } } @@ -524,48 +502,59 @@ func (si *bufferedIterator) Next() bool { if err != io.EOF { si.err = err } + si.Close() return false } - l, err := binary.ReadUvarint(si.s) + si.l, err = binary.ReadUvarint(si.s) if err != nil { if err != io.EOF { si.err = err - + si.Close() return false } } - for len(si.buf) < int(l) { + for len(si.buf) < int(si.l) { si.buf = append(si.buf, make([]byte, 1024)...) } - n, err := si.s.Read(si.buf[:l]) + n, err := si.s.Read(si.buf[:si.l]) if err != nil && err != io.EOF { si.err = err + si.Close() return false } - if n < int(l) { - _, err = si.s.Read(si.buf[n:l]) + if n < int(si.l) { + _, err = si.s.Read(si.buf[n:si.l]) if err != nil { si.err = err + si.Close() return false } } si.curT = ts - si.curLog = string(si.buf[:l]) - return true } func (si *bufferedIterator) Entry() logproto.Entry { return logproto.Entry{ Timestamp: time.Unix(0, si.curT), - Line: si.curLog, + Line: string(si.buf[:si.l]), } } -func (si *bufferedIterator) Error() error { return si.err } -func (si *bufferedIterator) Close() error { return si.err } +func (si *bufferedIterator) Error() error { return si.err } +func (si *bufferedIterator) Close() error { + if !si.closed { + si.closed = true + si.pool.PutReader(si.reader) + BufReaderPool.Put(si.s) + LineBufferPool.Put(si.buf) + IntBinBufferPool.Put(si.decBuf) + return si.err + } + return si.err +} func (si *bufferedIterator) Labels() string { return "" } diff --git a/pkg/chunkenc/gzip_test.go b/pkg/chunkenc/gzip_test.go index dc59559906f8..d23e5efd6781 100644 --- a/pkg/chunkenc/gzip_test.go +++ b/pkg/chunkenc/gzip_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "math" "math/rand" + "sync" "testing" "time" @@ -249,7 +250,7 @@ func BenchmarkReadGZIP(b *testing.B) { } i := int64(0) - for n := 0; n < 100; n++ { + for n := 0; n < 50; n++ { c := NewMemChunk(EncGZIP) // adds until full so we trigger cut which serialize using gzip for c.SpaceFor(entry) { @@ -259,16 +260,24 @@ func BenchmarkReadGZIP(b *testing.B) { } chunks = append(chunks, c) } + entries := []logproto.Entry{} for n := 0; n < b.N; n++ { + var wg sync.WaitGroup for _, c := range chunks { - iter, err := c.Iterator(time.Unix(0, 0), time.Unix(0, 100), logproto.BACKWARD) - if err != nil { - b.Fatal(err) - } - for iter.Next() { - - } + wg.Add(1) + go func(c Chunk) { + iterator, err := c.Iterator(time.Unix(0, 0), time.Unix(0, 10), logproto.BACKWARD) + if err != nil { + b.Fatal(err) + } + for iterator.Next() { + entries = append(entries, iterator.Entry()) + } + iterator.Close() + wg.Done() + }(c) } + wg.Wait() } } diff --git a/pkg/chunkenc/interface.go b/pkg/chunkenc/interface.go index afc1c0683f3a..dad84f539e91 100644 --- a/pkg/chunkenc/interface.go +++ b/pkg/chunkenc/interface.go @@ -63,9 +63,5 @@ type CompressionWriter interface { type CompressionReader interface { Read(p []byte) (int, error) Reset(r io.Reader) error -} - -type CompressionWriterPool interface { - Get() CompressionWriter - Put(CompressionWriter) + Close() error } diff --git a/pkg/chunkenc/lazy_chunk.go b/pkg/chunkenc/lazy_chunk.go index f23b404925a9..d6ef862514a0 100644 --- a/pkg/chunkenc/lazy_chunk.go +++ b/pkg/chunkenc/lazy_chunk.go @@ -52,6 +52,8 @@ type lazyIterator struct { from, through time.Time direction logproto.Direction context context.Context + + closed bool } func (it *lazyIterator) Next() bool { @@ -59,8 +61,16 @@ func (it *lazyIterator) Next() bool { return false } + if it.closed { + return false + } + if it.EntryIterator != nil { - return it.EntryIterator.Next() + next := it.EntryIterator.Next() + if !next { + it.Close() + } + return next } chk, err := it.chunk.getChunk(it.context) @@ -70,7 +80,6 @@ func (it *lazyIterator) Next() bool { } it.EntryIterator, it.err = chk.Iterator(it.from, it.through, it.direction) - return it.Next() } @@ -82,6 +91,19 @@ func (it *lazyIterator) Error() error { if it.err != nil { return it.err } + if it.EntryIterator != nil { + return it.EntryIterator.Error() + } + return nil +} - return it.EntryIterator.Error() +func (it *lazyIterator) Close() error { + if it.EntryIterator != nil { + it.chunk.Chunk.Data = nil + it.closed = true + err := it.EntryIterator.Close() + it.EntryIterator = nil + return err + } + return nil } diff --git a/pkg/chunkenc/pool.go b/pkg/chunkenc/pool.go new file mode 100644 index 000000000000..8ca041ea346f --- /dev/null +++ b/pkg/chunkenc/pool.go @@ -0,0 +1,112 @@ +package chunkenc + +import ( + "bufio" + "bytes" + "compress/gzip" + "encoding/binary" + "io" + "sync" +) + +type CompressionPool interface { + GetWriter(io.Writer) CompressionWriter + PutWriter(CompressionWriter) + GetReader(io.Reader) CompressionReader + PutReader(CompressionReader) +} + +var ( + Gzip GzipPool + BufReaderPool = &BufioReaderPool{ + pool: sync.Pool{ + New: func() interface{} { return bufio.NewReader(nil) }, + }, + } + ByteBufferPool = sync.Pool{ + // New is called when a new instance is needed + New: func() interface{} { + return new(bytes.Buffer) + }, + } + LineBufferPool = newBufferPoolWithSize(1024) + IntBinBufferPool = newBufferPoolWithSize(binary.MaxVarintLen64) +) + +type GzipPool struct { + readers sync.Pool + writers sync.Pool +} + +func (pool *GzipPool) GetReader(src io.Reader) (reader CompressionReader) { + if r := pool.readers.Get(); r != nil { + reader = r.(CompressionReader) + err := reader.Reset(src) + if err != nil { + panic(err) + } + } else { + var err error + reader, err = gzip.NewReader(src) + if err != nil { + panic(err) + } + } + return reader +} + +func (pool *GzipPool) PutReader(reader CompressionReader) { + pool.readers.Put(reader) +} + +func (pool *GzipPool) GetWriter(dst io.Writer) (writer CompressionWriter) { + if w := pool.writers.Get(); w != nil { + writer = w.(CompressionWriter) + writer.Reset(dst) + } else { + writer = gzip.NewWriter(dst) + } + return writer +} + +func (pool *GzipPool) PutWriter(writer CompressionWriter) { + pool.writers.Put(writer) +} + +// BufioReaderPool is a bufio reader that uses sync.Pool. +type BufioReaderPool struct { + pool sync.Pool +} + +// Get returns a bufio.Reader which reads from r. The buffer size is that of the pool. +func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader { + buf := bufPool.pool.Get().(*bufio.Reader) + buf.Reset(r) + return buf +} + +// Put puts the bufio.Reader back into the pool. +func (bufPool *BufioReaderPool) Put(b *bufio.Reader) { + b.Reset(nil) + bufPool.pool.Put(b) +} + +type bufferPool struct { + pool sync.Pool +} + +func newBufferPoolWithSize(size int) *bufferPool { + return &bufferPool{ + pool: sync.Pool{ + New: func() interface{} { return make([]byte, size) }, + }, + } +} + +func (bp *bufferPool) Get() []byte { + return bp.pool.Get().([]byte) +} + +func (bp *bufferPool) Put(b []byte) { + bp.pool.Put(b) +} diff --git a/pkg/iter/iterator.go b/pkg/iter/iterator.go index fe18b7c0cae4..ba95cc9365a1 100644 --- a/pkg/iter/iterator.go +++ b/pkg/iter/iterator.go @@ -113,6 +113,10 @@ type heapIterator struct { heap.Interface Peek() EntryIterator } + is []EntryIterator + prenext bool + + tuples tuples currEntry logproto.Entry currLabels string errs []error @@ -121,7 +125,7 @@ type heapIterator struct { // NewHeapIterator returns a new iterator which uses a heap to merge together // entries for multiple interators. func NewHeapIterator(is []EntryIterator, direction logproto.Direction) HeapIterator { - result := &heapIterator{} + result := &heapIterator{is: is} switch direction { case logproto.BACKWARD: result.heap = &iteratorMaxHeap{} @@ -131,11 +135,7 @@ func NewHeapIterator(is []EntryIterator, direction logproto.Direction) HeapItera panic("bad direction") } - // pre-next each iterator, drop empty. - for _, i := range is { - result.requeue(i, false) - } - + result.tuples = make([]tuple, 0, len(is)) return result } @@ -160,7 +160,20 @@ type tuple struct { EntryIterator } +type tuples []tuple + +func (t tuples) Len() int { return len(t) } +func (t tuples) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t tuples) Less(i, j int) bool { return t[i].Line < t[j].Line } + func (i *heapIterator) Next() bool { + if !i.prenext { + i.prenext = true + // pre-next each iterator, drop empty. + for _, it := range i.is { + i.requeue(it, false) + } + } if i.heap.Len() == 0 { return false } @@ -170,16 +183,15 @@ func (i *heapIterator) Next() bool { // heap with the same timestamp, and pop the ones whose common value // occurs most often. - tuples := make([]tuple, 0, i.heap.Len()) for i.heap.Len() > 0 { next := i.heap.Peek() entry := next.Entry() - if len(tuples) > 0 && (tuples[0].Labels() != next.Labels() || !tuples[0].Timestamp.Equal(entry.Timestamp)) { + if len(i.tuples) > 0 && (i.tuples[0].Labels() != next.Labels() || !i.tuples[0].Timestamp.Equal(entry.Timestamp)) { break } heap.Pop(i.heap) - tuples = append(tuples, tuple{ + i.tuples = append(i.tuples, tuple{ Entry: entry, EntryIterator: next, }) @@ -187,22 +199,20 @@ func (i *heapIterator) Next() bool { // Find in entry which occurs most often which, due to quorum based // replication, is guaranteed to be the correct next entry. - t := mostCommon(tuples) + t := mostCommon(i.tuples) i.currEntry = t.Entry i.currLabels = t.Labels() // Requeue the iterators, advancing them if they were consumed. - for j := range tuples { - i.requeue(tuples[j].EntryIterator, tuples[j].Line != i.currEntry.Line) + for j := range i.tuples { + i.requeue(i.tuples[j].EntryIterator, i.tuples[j].Line != i.currEntry.Line) } - + i.tuples = i.tuples[:0] return true } -func mostCommon(tuples []tuple) tuple { - sort.Slice(tuples, func(i, j int) bool { - return tuples[i].Line < tuples[j].Line - }) +func mostCommon(tuples tuples) tuple { + sort.Sort(tuples) result := tuples[0] count, max := 0, 0 for i := 0; i < len(tuples)-1; i++ { @@ -247,6 +257,7 @@ func (i *heapIterator) Close() error { return err } } + i.tuples = nil return nil } @@ -353,10 +364,13 @@ func NewNonOverlappingIterator(iterators []EntryIterator, labels string) EntryIt func (i *nonOverlappingIterator) Next() bool { for i.curr == nil || !i.curr.Next() { + if i.curr != nil { + // close the previous iterator if any + i.curr.Close() + } if i.i >= len(i.iterators) { return false } - i.curr = i.iterators[i.i] i.i++ } @@ -377,7 +391,10 @@ func (i *nonOverlappingIterator) Labels() string { } func (i *nonOverlappingIterator) Error() error { - return i.curr.Error() + if i.curr != nil { + return i.curr.Error() + } + return nil } func (i *nonOverlappingIterator) Close() error { @@ -415,23 +432,32 @@ func (i *timeRangedIterator) Next() bool { } type entryIteratorBackward struct { - cur logproto.Entry - entries []logproto.Entry + forwardIter EntryIterator + cur logproto.Entry + entries []logproto.Entry + loaded bool } // NewEntryIteratorBackward returns an iterator which loads all the entries // of an existing iterator, and then iterates over them backward. func NewEntryIteratorBackward(it EntryIterator) (EntryIterator, error) { - entries := make([]logproto.Entry, 0, 128) - for it.Next() { - entries = append(entries, it.Entry()) - } + return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 128), forwardIter: it}, it.Error() +} - return &entryIteratorBackward{entries: entries}, it.Error() +func (i *entryIteratorBackward) load() { + if !i.loaded { + i.loaded = true + for i.forwardIter.Next() { + i.entries = append(i.entries, i.forwardIter.Entry()) + } + i.forwardIter.Close() + } } func (i *entryIteratorBackward) Next() bool { + i.load() if len(i.entries) == 0 { + i.entries = nil return false } diff --git a/pkg/storage/hack/main.go b/pkg/storage/hack/main.go new file mode 100644 index 000000000000..e09dc9ac7428 --- /dev/null +++ b/pkg/storage/hack/main.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "fmt" + "log" + "math/rand" + "sync" + "time" + + "github.com/cortexproject/cortex/pkg/chunk" + "github.com/cortexproject/cortex/pkg/chunk/local" + "github.com/cortexproject/cortex/pkg/chunk/storage" + "github.com/cortexproject/cortex/pkg/ingester/client" + "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/grafana/loki/pkg/chunkenc" + "github.com/grafana/loki/pkg/logproto" + lstore "github.com/grafana/loki/pkg/storage" + "github.com/grafana/loki/pkg/util" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/weaveworks/common/user" +) + +var start = model.Time(1523750400000) + +var ctx = user.InjectOrgID(context.Background(), "fake") + +func main() { + // fill up the store with 1gib of data to run benchmark + fillStore() +} + +func getStore() (lstore.Store, error) { + store, err := lstore.NewStore(storage.Config{ + BoltDBConfig: local.BoltDBConfig{Directory: "/tmp/benchmark/index"}, + FSConfig: local.FSConfig{Directory: "/tmp/benchmark/chunks"}, + }, chunk.StoreConfig{}, chunk.SchemaConfig{ + Configs: []chunk.PeriodConfig{ + chunk.PeriodConfig{ + From: chunk.DayTime{Time: start}, + IndexType: "boltdb", + ObjectType: "filesystem", + Schema: "v9", + IndexTables: chunk.PeriodicTableConfig{ + Prefix: "index_", + Period: time.Hour * 168, + }, + }, + }, + }, &validation.Overrides{}) + if err != nil { + return nil, err + } + return store, nil +} + +func fillStore() error { + + store, err := getStore() + if err != nil { + return err + } + defer store.Stop() + + var wgPush sync.WaitGroup + var flushCount int + // insert 5 streams with a random logs every seconds for an hour + // the string is randomize so chunks are big ~2mb + // take ~2min to build 1gib of data + for i := 0; i < 5; i++ { + wgPush.Add(1) + go func(j int) { + defer wgPush.Done() + lbs, err := util.ToClientLabels(fmt.Sprintf("{foo=\"bar\",level=\"%d\"}", j)) + if err != nil { + panic(err) + } + labelsBuilder := labels.NewBuilder(client.FromLabelAdaptersToLabels(lbs)) + labelsBuilder.Set(labels.MetricName, "logs") + metric := labelsBuilder.Labels() + fp := client.FastFingerprint(lbs) + chunkEnc := chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144) + for ts := start.UnixNano(); ts < start.UnixNano()+time.Hour.Nanoseconds(); ts = ts + time.Millisecond.Nanoseconds() { + entry := &logproto.Entry{ + Timestamp: time.Unix(0, ts), + Line: randString(250), + } + if chunkEnc.SpaceFor(entry) { + chunkEnc.Append(entry) + } else { + from, to := chunkEnc.Bounds() + c := chunk.NewChunk("fake", fp, metric, chunkenc.NewFacade(chunkEnc), model.TimeFromUnixNano(from.UnixNano()), model.TimeFromUnixNano(to.UnixNano())) + if err := c.Encode(); err != nil { + panic(err) + } + err := store.Put(ctx, []chunk.Chunk{c}) + if err != nil { + panic(err) + } + flushCount++ + log.Println("flushed ", flushCount, from.UnixNano(), to.UnixNano(), metric) + if flushCount >= 600 { + // 600 chunk is enough data ~1gib + return + } + chunkEnc = chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144) + } + } + + }(i) + + } + wgPush.Wait() + return nil +} + +const charset = "abcdefghijklmnopqrstuvwxyz" + + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func randStringWithCharset(length int, charset string) string { + b := make([]byte, length) + for i := range b { + b[i] = charset[rand.Intn(len(charset)-1)] + } + return string(b) +} + +func randString(length int) string { + return randStringWithCharset(length, charset) +} diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go new file mode 100644 index 000000000000..faeceea56f30 --- /dev/null +++ b/pkg/storage/store_test.go @@ -0,0 +1,82 @@ +package storage + +import ( + "context" + "log" + "runtime" + "testing" + "time" + + "github.com/cortexproject/cortex/pkg/chunk" + "github.com/cortexproject/cortex/pkg/chunk/local" + "github.com/cortexproject/cortex/pkg/chunk/storage" + "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/grafana/loki/pkg/logproto" + "github.com/prometheus/common/model" + "github.com/weaveworks/common/user" +) + +var ( + start = model.Time(1523750400000) + m runtime.MemStats + ctx = user.InjectOrgID(context.Background(), "fake") +) + +//go test -bench=. -benchmem -memprofile memprofile.out -cpuprofile profile.out +func Benchmark_store_LazyQuery(b *testing.B) { + + store, err := getStore() + if err != nil { + b.Fatal(err) + } + for i := 0; i < b.N; i++ { + iter, err := store.LazyQuery(ctx, &logproto.QueryRequest{ + Query: "{foo=\"bar\"}", + Regex: "fizz", + Limit: 1000, + Start: time.Unix(0, start.UnixNano()), + End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), + Direction: logproto.BACKWARD, + }) + if err != nil { + b.Fatal(err) + } + res := []logproto.Entry{} + printHeap() + for iter.Next() { + printHeap() + res = append(res, iter.Entry()) + } + iter.Close() + printHeap() + } +} + +func printHeap() { + runtime.ReadMemStats(&m) + log.Printf("HeapInuse: %d Mbytes\n", m.HeapInuse/1024/1024) +} + +func getStore() (Store, error) { + store, err := NewStore(storage.Config{ + BoltDBConfig: local.BoltDBConfig{Directory: "/tmp/benchmark/index"}, + FSConfig: local.FSConfig{Directory: "/tmp/benchmark/chunks"}, + }, chunk.StoreConfig{}, chunk.SchemaConfig{ + Configs: []chunk.PeriodConfig{ + chunk.PeriodConfig{ + From: chunk.DayTime{Time: start}, + IndexType: "boltdb", + ObjectType: "filesystem", + Schema: "v9", + IndexTables: chunk.PeriodicTableConfig{ + Prefix: "index_", + Period: time.Hour * 168, + }, + }, + }, + }, &validation.Overrides{}) + if err != nil { + return nil, err + } + return store, nil +} From 2945d774e6b845002a729971c6e0062528352f31 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 27 Jun 2019 00:42:51 -0400 Subject: [PATCH 06/25] improve test, add timeout in cortex --- pkg/storage/store_test.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index faeceea56f30..c5534f3a952d 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -25,14 +25,14 @@ var ( //go test -bench=. -benchmem -memprofile memprofile.out -cpuprofile profile.out func Benchmark_store_LazyQuery(b *testing.B) { - store, err := getStore() - if err != nil { - b.Fatal(err) - } for i := 0; i < b.N; i++ { + store, err := getStore() + if err != nil { + b.Fatal(err) + } iter, err := store.LazyQuery(ctx, &logproto.QueryRequest{ Query: "{foo=\"bar\"}", - Regex: "fizz", + Regex: "fuzz", Limit: 1000, Start: time.Unix(0, start.UnixNano()), End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), @@ -42,19 +42,27 @@ func Benchmark_store_LazyQuery(b *testing.B) { b.Fatal(err) } res := []logproto.Entry{} - printHeap() + printHeap(b) + j := 0 for iter.Next() { - printHeap() + j++ + printHeap(b) res = append(res, iter.Entry()) + // todo this should be done in the store. + if j == 1000 { + break + } } iter.Close() - printHeap() + printHeap(b) + log.Println("line fetched", len(res)) + store.Stop() } } -func printHeap() { +func printHeap(b *testing.B) { runtime.ReadMemStats(&m) - log.Printf("HeapInuse: %d Mbytes\n", m.HeapInuse/1024/1024) + log.Printf("Benchmark %d HeapInuse: %d Mbytes\n", b.N, m.HeapInuse/1024/1024) } func getStore() (Store, error) { From 5657c63cfdc32810a784375c951ba73d365ac0a8 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Sat, 29 Jun 2019 23:01:50 -0400 Subject: [PATCH 07/25] better perf with benchmark on store --- Gopkg.lock | 5 +- Makefile | 6 ++ pkg/chunkenc/dumb_chunk.go | 11 +-- pkg/chunkenc/facade.go | 2 +- pkg/chunkenc/gzip.go | 90 +++++++++++++--------- pkg/chunkenc/gzip_test.go | 4 +- pkg/chunkenc/interface.go | 3 +- pkg/chunkenc/lazy_chunk.go | 21 ++++-- pkg/chunkenc/pool.go | 32 +------- pkg/iter/iterator.go | 79 +++++++++++++------ pkg/logql/ast.go | 45 ++++++----- pkg/storage/hack/main.go | 26 ++++--- pkg/storage/store.go | 22 +++--- pkg/storage/store_test.go | 125 ++++++++++++++++++++++++------- production/helm/loki/values.yaml | 3 +- 15 files changed, 302 insertions(+), 172 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c2a8443715f9..ff0ccba0eb3f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1563,6 +1563,7 @@ "github.com/bmatcuk/doublestar", "github.com/cortexproject/cortex/pkg/chunk", "github.com/cortexproject/cortex/pkg/chunk/encoding", + "github.com/cortexproject/cortex/pkg/chunk/local", "github.com/cortexproject/cortex/pkg/chunk/storage", "github.com/cortexproject/cortex/pkg/ingester/client", "github.com/cortexproject/cortex/pkg/ingester/index", @@ -1573,6 +1574,7 @@ "github.com/docker/docker/api/types/plugins/logdriver", "github.com/docker/docker/daemon/logger", "github.com/docker/docker/daemon/logger/jsonfilelog", + "github.com/docker/docker/daemon/logger/templates", "github.com/docker/docker/pkg/ioutils", "github.com/docker/go-plugins-helpers/sdk", "github.com/fatih/color", @@ -1611,12 +1613,12 @@ "github.com/shurcooL/httpfs/filter", "github.com/shurcooL/httpfs/union", "github.com/shurcooL/vfsgen", - "github.com/sirupsen/logrus", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/require", "github.com/tonistiigi/fifo", "github.com/weaveworks/common/httpgrpc", "github.com/weaveworks/common/httpgrpc/server", + "github.com/weaveworks/common/logging", "github.com/weaveworks/common/middleware", "github.com/weaveworks/common/server", "github.com/weaveworks/common/tracing", @@ -1624,6 +1626,7 @@ "golang.org/x/net/context", "google.golang.org/grpc", "google.golang.org/grpc/health/grpc_health_v1", + "google.golang.org/grpc/metadata", "gopkg.in/alecthomas/kingpin.v2", "gopkg.in/fsnotify.v1", "gopkg.in/yaml.v2", diff --git a/Makefile b/Makefile index ff74cc07b6a2..09ae801c648f 100644 --- a/Makefile +++ b/Makefile @@ -312,3 +312,9 @@ push-plugin: build-plugin enable-plugin: docker plugin enable grafana/loki-docker-driver:$(PLUGIN_TAG) + +fill-store: + go run ./pkg/storage/hack/main.go + +benchmark-store: fill-store + go test ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out \ No newline at end of file diff --git a/pkg/chunkenc/dumb_chunk.go b/pkg/chunkenc/dumb_chunk.go index 8393c771655a..9c31ca0a9a9e 100644 --- a/pkg/chunkenc/dumb_chunk.go +++ b/pkg/chunkenc/dumb_chunk.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql" ) const ( @@ -18,7 +19,7 @@ func NewDumbChunk() Chunk { } type dumbChunk struct { - entries []logproto.Entry + entries []*logproto.Entry } func (c *dumbChunk) Bounds() (time.Time, time.Time) { @@ -41,7 +42,7 @@ func (c *dumbChunk) Append(entry *logproto.Entry) error { return ErrOutOfOrder } - c.entries = append(c.entries, *entry) + c.entries = append(c.entries, entry) return nil } @@ -51,7 +52,7 @@ func (c *dumbChunk) Size() int { // Returns an iterator that goes from _most_ recent to _least_ recent (ie, // backwards). -func (c *dumbChunk) Iterator(from, through time.Time, direction logproto.Direction) (iter.EntryIterator, error) { +func (c *dumbChunk) Iterator(from, through time.Time, direction logproto.Direction, filter logql.Filter) (iter.EntryIterator, error) { i := sort.Search(len(c.entries), func(i int) bool { return !from.After(c.entries[i].Timestamp) }) @@ -83,7 +84,7 @@ func (c *dumbChunk) Bytes() ([]byte, error) { type dumbChunkIterator struct { direction logproto.Direction i int - entries []logproto.Entry + entries []*logproto.Entry } func (i *dumbChunkIterator) Next() bool { @@ -99,7 +100,7 @@ func (i *dumbChunkIterator) Next() bool { } } -func (i *dumbChunkIterator) Entry() logproto.Entry { +func (i *dumbChunkIterator) Entry() *logproto.Entry { return i.entries[i.i] } diff --git a/pkg/chunkenc/facade.go b/pkg/chunkenc/facade.go index ef3801d1dc22..eb8820ff1b51 100644 --- a/pkg/chunkenc/facade.go +++ b/pkg/chunkenc/facade.go @@ -12,7 +12,7 @@ const GzipLogChunk = encoding.Encoding(128) func init() { encoding.MustRegisterEncoding(GzipLogChunk, "GzipLogChunk", func() encoding.Chunk { return &Facade{ - c: NewMemChunk(EncGZIP), + c: NewMemChunk(EncGZIP, false), } }) } diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 39c7a14c6f8a..14c3924cf2af 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -7,10 +7,10 @@ import ( "hash" "hash/crc32" "io" - "math" "time" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql" "github.com/grafana/loki/pkg/iter" @@ -129,19 +129,20 @@ type entry struct { // NewMemChunkSize returns a new in-mem chunk. // Mainly for config push size. -func NewMemChunkSize(enc Encoding, blockSize int) *MemChunk { +func NewMemChunkSize(enc Encoding, blockSize int, head bool) *MemChunk { c := &MemChunk{ blockSize: blockSize, // The blockSize in bytes. blocks: []block{}, - head: &headBlock{ - mint: math.MaxInt64, - maxt: math.MinInt64, - }, + head: nil, encoding: enc, } + if head { + c.head = &headBlock{} + } + switch enc { case EncGZIP: c.cPool = &Gzip @@ -153,15 +154,15 @@ func NewMemChunkSize(enc Encoding, blockSize int) *MemChunk { } // NewMemChunk returns a new in-mem chunk for query. -func NewMemChunk(enc Encoding) *MemChunk { - return NewMemChunkSize(enc, 256*1024) +func NewMemChunk(enc Encoding, head bool) *MemChunk { + return NewMemChunkSize(enc, 256*1024, head) } // NewByteChunk returns a MemChunk on the passed bytes. func NewByteChunk(b []byte) (*MemChunk, error) { bc := &MemChunk{ cPool: &Gzip, - head: &headBlock{}, // Dummy, empty headblock. + head: nil, // Dummy, empty headblock. } db := decbuf{b: b} @@ -189,6 +190,7 @@ func NewByteChunk(b []byte) (*MemChunk, error) { // Read the number of blocks. num := db.uvarint() + bc.blocks = make([]block, num) for i := 0; i < num; i++ { blk := block{} @@ -303,7 +305,7 @@ func (c *MemChunk) Size() int { ne += blk.numEntries } - if !c.head.isEmpty() { + if c.head != nil && !c.head.isEmpty() { ne += len(c.head.entries) } @@ -367,7 +369,7 @@ func (c *MemChunk) Bounds() (fromT, toT time.Time) { to = c.blocks[len(c.blocks)-1].maxt } - if !c.head.isEmpty() { + if c.head != nil && !c.head.isEmpty() { if from == 0 || from > c.head.mint { from = c.head.mint } @@ -381,22 +383,30 @@ func (c *MemChunk) Bounds() (fromT, toT time.Time) { } // Iterator implements Chunk. -func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction) (iter.EntryIterator, error) { +func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction, filter logql.Filter) (iter.EntryIterator, error) { mint, maxt := mintT.UnixNano(), maxtT.UnixNano() - its := make([]iter.EntryIterator, 0, len(c.blocks)) + its := make([]iter.EntryIterator, 0, len(c.blocks)+1) for _, b := range c.blocks { if maxt > b.mint && b.maxt > mint { - it, err := b.iterator(c.cPool) + it, err := b.iterator(c.cPool, filter) if err != nil { return nil, err } - its = append(its, it) } } - its = append(its, c.head.iterator(mint, maxt)) + if c.head != nil { + // todo filter + its = append(its, c.head.iterator(mint, maxt)) + } + + if direction == logproto.FORWARD { + for i, j := 0, len(its)-1; i < j; i, j = i+1, j-1 { + its[i], its[j] = its[j], its[i] + } + } iterForward := iter.NewTimeRangedIterator( iter.NewNonOverlappingIterator(its, ""), @@ -411,11 +421,11 @@ func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction return iter.NewEntryIteratorBackward(iterForward) } -func (b block) iterator(pool CompressionPool) (iter.EntryIterator, error) { +func (b block) iterator(pool CompressionPool, filter logql.Filter) (iter.EntryIterator, error) { if len(b.b) == 0 { return emptyIterator, nil } - return newBufferedIterator(pool, b.b), nil + return newBufferedIterator(pool, b.b, filter), nil } func (hb *headBlock) iterator(mint, maxt int64) iter.EntryIterator { @@ -455,8 +465,8 @@ func (li *listIterator) Next() bool { return false } -func (li *listIterator) Entry() logproto.Entry { - return logproto.Entry{ +func (li *listIterator) Entry() *logproto.Entry { + return &logproto.Entry{ Timestamp: time.Unix(0, li.cur.t), Line: li.cur.s, } @@ -471,8 +481,8 @@ type bufferedIterator struct { reader CompressionReader pool CompressionPool - curT int64 - l uint64 + cur logproto.Entry + l uint64 err error @@ -480,19 +490,21 @@ type bufferedIterator struct { decBuf []byte // The buffer for decoding the lengths. closed bool -} -func newBufferedIterator(pool CompressionPool, b []byte) *bufferedIterator { - buf := bytes.NewBuffer(b) + filter logql.Filter +} - r := pool.GetReader(buf) +var lineParsed int64 +func newBufferedIterator(pool CompressionPool, b []byte, filter logql.Filter) *bufferedIterator { + r := pool.GetReader(bytes.NewBuffer(b)) return &bufferedIterator{ s: BufReaderPool.Get(r), reader: r, pool: pool, - buf: LineBufferPool.Get(), - decBuf: IntBinBufferPool.Get(), + filter: filter, + buf: make([]byte, 256), + decBuf: make([]byte, binary.MaxVarintLen64), } } @@ -516,7 +528,7 @@ func (si *bufferedIterator) Next() bool { } for len(si.buf) < int(si.l) { - si.buf = append(si.buf, make([]byte, 1024)...) + si.buf = append(si.buf, make([]byte, 256)...) } n, err := si.s.Read(si.buf[:si.l]) @@ -534,15 +546,17 @@ func (si *bufferedIterator) Next() bool { } } - si.curT = ts + if si.filter != nil && !si.filter(si.buf[:si.l]) { + return si.Next() + } + si.cur.Line = string(si.buf[:si.l]) + si.cur.Timestamp = time.Unix(0, ts) return true } -func (si *bufferedIterator) Entry() logproto.Entry { - return logproto.Entry{ - Timestamp: time.Unix(0, si.curT), - Line: string(si.buf[:si.l]), - } +func (si *bufferedIterator) Entry() *logproto.Entry { + // log.Println("lineParsed", atomic.AddInt64(&lineParsed, 1)) + return &si.cur } func (si *bufferedIterator) Error() error { return si.err } @@ -551,8 +565,10 @@ func (si *bufferedIterator) Close() error { si.closed = true si.pool.PutReader(si.reader) BufReaderPool.Put(si.s) - LineBufferPool.Put(si.buf) - IntBinBufferPool.Put(si.decBuf) + si.s = nil + si.buf = nil + si.decBuf = nil + si.reader = nil return si.err } return si.err diff --git a/pkg/chunkenc/gzip_test.go b/pkg/chunkenc/gzip_test.go index d23e5efd6781..d4c363e6d30c 100644 --- a/pkg/chunkenc/gzip_test.go +++ b/pkg/chunkenc/gzip_test.go @@ -229,7 +229,7 @@ func BenchmarkWriteGZIP(b *testing.B) { i := int64(0) for n := 0; n < b.N; n++ { - c := NewMemChunk(EncGZIP) + c := NewMemChunk(EncGZIP, true) // adds until full so we trigger cut which serialize using gzip for c.SpaceFor(entry) { c.Append(entry) @@ -251,7 +251,7 @@ func BenchmarkReadGZIP(b *testing.B) { i := int64(0) for n := 0; n < 50; n++ { - c := NewMemChunk(EncGZIP) + c := NewMemChunk(EncGZIP, true) // adds until full so we trigger cut which serialize using gzip for c.SpaceFor(entry) { c.Append(entry) diff --git a/pkg/chunkenc/interface.go b/pkg/chunkenc/interface.go index dad84f539e91..3b49b5e2c4e3 100644 --- a/pkg/chunkenc/interface.go +++ b/pkg/chunkenc/interface.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql" ) // Errors returned by the chunk interface. @@ -46,7 +47,7 @@ type Chunk interface { Bounds() (time.Time, time.Time) SpaceFor(*logproto.Entry) bool Append(*logproto.Entry) error - Iterator(from, through time.Time, direction logproto.Direction) (iter.EntryIterator, error) + Iterator(from, through time.Time, direction logproto.Direction, filter logql.Filter) (iter.EntryIterator, error) Size() int Bytes() ([]byte, error) } diff --git a/pkg/chunkenc/lazy_chunk.go b/pkg/chunkenc/lazy_chunk.go index d6ef862514a0..299b937da171 100644 --- a/pkg/chunkenc/lazy_chunk.go +++ b/pkg/chunkenc/lazy_chunk.go @@ -7,6 +7,7 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql" ) // LazyChunk loads the chunk when it is accessed. @@ -26,15 +27,16 @@ func (c *LazyChunk) getChunk(ctx context.Context) (Chunk, error) { } // Iterator returns an entry iterator. -func (c LazyChunk) Iterator(ctx context.Context, from, through time.Time, direction logproto.Direction) (iter.EntryIterator, error) { +func (c *LazyChunk) Iterator(ctx context.Context, from, through time.Time, direction logproto.Direction, filter logql.Filter) (iter.EntryIterator, error) { // If the chunk is already loaded, then use that. if c.Chunk.Data != nil { lokiChunk := c.Chunk.Data.(*Facade).LokiChunk() - return lokiChunk.Iterator(from, through, direction) + return lokiChunk.Iterator(from, through, direction, filter) } return &lazyIterator{ - chunk: c, + chunk: c, + filter: filter, from: from, through: through, @@ -46,16 +48,20 @@ func (c LazyChunk) Iterator(ctx context.Context, from, through time.Time, direct type lazyIterator struct { iter.EntryIterator - chunk LazyChunk + chunk *LazyChunk err error from, through time.Time direction logproto.Direction context context.Context + filter logql.Filter closed bool } +var chunksOpen int64 +var chunksTotal int64 + func (it *lazyIterator) Next() bool { if it.err != nil { return false @@ -78,8 +84,8 @@ func (it *lazyIterator) Next() bool { it.err = err return false } - - it.EntryIterator, it.err = chk.Iterator(it.from, it.through, it.direction) + it.EntryIterator, it.err = chk.Iterator(it.from, it.through, it.direction, it.filter) + it.chunk = nil return it.Next() } @@ -99,7 +105,8 @@ func (it *lazyIterator) Error() error { func (it *lazyIterator) Close() error { if it.EntryIterator != nil { - it.chunk.Chunk.Data = nil + // log.Println("Chunk Open", atomic.AddInt64(&chunksOpen, -1)) + it.chunk = nil it.closed = true err := it.EntryIterator.Close() it.EntryIterator = nil diff --git a/pkg/chunkenc/pool.go b/pkg/chunkenc/pool.go index 8ca041ea346f..e1da7ed9645d 100644 --- a/pkg/chunkenc/pool.go +++ b/pkg/chunkenc/pool.go @@ -2,9 +2,8 @@ package chunkenc import ( "bufio" - "bytes" "compress/gzip" - "encoding/binary" + "io" "sync" ) @@ -23,14 +22,6 @@ var ( New: func() interface{} { return bufio.NewReader(nil) }, }, } - ByteBufferPool = sync.Pool{ - // New is called when a new instance is needed - New: func() interface{} { - return new(bytes.Buffer) - }, - } - LineBufferPool = newBufferPoolWithSize(1024) - IntBinBufferPool = newBufferPoolWithSize(binary.MaxVarintLen64) ) type GzipPool struct { @@ -87,26 +78,5 @@ func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader { // Put puts the bufio.Reader back into the pool. func (bufPool *BufioReaderPool) Put(b *bufio.Reader) { - b.Reset(nil) bufPool.pool.Put(b) } - -type bufferPool struct { - pool sync.Pool -} - -func newBufferPoolWithSize(size int) *bufferPool { - return &bufferPool{ - pool: sync.Pool{ - New: func() interface{} { return make([]byte, size) }, - }, - } -} - -func (bp *bufferPool) Get() []byte { - return bp.pool.Get().([]byte) -} - -func (bp *bufferPool) Put(b []byte) { - bp.pool.Put(b) -} diff --git a/pkg/iter/iterator.go b/pkg/iter/iterator.go index ba95cc9365a1..309fa98487dd 100644 --- a/pkg/iter/iterator.go +++ b/pkg/iter/iterator.go @@ -14,7 +14,7 @@ import ( // EntryIterator iterates over entries in time-order. type EntryIterator interface { Next() bool - Entry() logproto.Entry + Entry() *logproto.Entry Labels() string Error() error Close() error @@ -49,8 +49,8 @@ func (i *streamIterator) Labels() string { return i.labels } -func (i *streamIterator) Entry() logproto.Entry { - return i.entries[i.i] +func (i *streamIterator) Entry() *logproto.Entry { + return &i.entries[i.i] } func (i *streamIterator) Close() error { @@ -117,7 +117,7 @@ type heapIterator struct { prenext bool tuples tuples - currEntry logproto.Entry + currEntry *logproto.Entry currLabels string errs []error } @@ -156,7 +156,7 @@ func (i *heapIterator) Push(ei EntryIterator) { } type tuple struct { - logproto.Entry + *logproto.Entry EntryIterator } @@ -173,6 +173,7 @@ func (i *heapIterator) Next() bool { for _, it := range i.is { i.requeue(it, false) } + i.is = nil } if i.heap.Len() == 0 { return false @@ -232,7 +233,7 @@ func mostCommon(tuples tuples) tuple { return result } -func (i *heapIterator) Entry() logproto.Entry { +func (i *heapIterator) Entry() *logproto.Entry { return i.currEntry } @@ -309,7 +310,7 @@ func (i *queryClientIterator) Next() bool { return true } -func (i *queryClientIterator) Entry() logproto.Entry { +func (i *queryClientIterator) Entry() *logproto.Entry { return i.curr.Entry() } @@ -328,6 +329,8 @@ func (i *queryClientIterator) Close() error { type filter struct { EntryIterator f func(string) bool + + curr *logproto.Entry } // NewFilter builds a filtering iterator. @@ -340,13 +343,20 @@ func NewFilter(f func(string) bool, i EntryIterator) EntryIterator { func (i *filter) Next() bool { for i.EntryIterator.Next() { - if i.f(i.Entry().Line) { + curr := i.EntryIterator.Entry() + if i.f(curr.Line) { + i.curr = curr return true } } + i.EntryIterator.Close() return false } +func (i *filter) Entry() *logproto.Entry { + return i.curr +} + type nonOverlappingIterator struct { labels string i int @@ -364,21 +374,25 @@ func NewNonOverlappingIterator(iterators []EntryIterator, labels string) EntryIt func (i *nonOverlappingIterator) Next() bool { for i.curr == nil || !i.curr.Next() { + if len(i.iterators) == 0 { + if i.curr != nil { + i.curr.Close() + i.curr = nil + } + return false + } if i.curr != nil { - // close the previous iterator if any i.curr.Close() + i.curr = nil } - if i.i >= len(i.iterators) { - return false - } - i.curr = i.iterators[i.i] i.i++ + i.curr, i.iterators = i.iterators[0], i.iterators[1:] } return true } -func (i *nonOverlappingIterator) Entry() logproto.Entry { +func (i *nonOverlappingIterator) Entry() *logproto.Entry { return i.curr.Entry() } @@ -398,6 +412,11 @@ func (i *nonOverlappingIterator) Error() error { } func (i *nonOverlappingIterator) Close() error { + for _, iter := range i.iterators { + iter.Close() + } + i.iterators = nil + i.curr = nil return nil } @@ -417,7 +436,10 @@ func NewTimeRangedIterator(it EntryIterator, mint, maxt time.Time) EntryIterator func (i *timeRangedIterator) Next() bool { ok := i.EntryIterator.Next() - + if !ok { + i.EntryIterator.Close() + return ok + } ts := i.EntryIterator.Entry().Timestamp for ok && i.mint.After(ts) { ok = i.EntryIterator.Next() @@ -427,21 +449,31 @@ func (i *timeRangedIterator) Next() bool { if ok && (i.maxt.Before(ts) || i.maxt.Equal(ts)) { // The maxt is exclusive. ok = false } - + if !ok { + i.EntryIterator.Close() + } return ok } type entryIteratorBackward struct { forwardIter EntryIterator - cur logproto.Entry - entries []logproto.Entry + cur *logproto.Entry + entries []*logproto.Entry loaded bool + + forward bool } // NewEntryIteratorBackward returns an iterator which loads all the entries // of an existing iterator, and then iterates over them backward. func NewEntryIteratorBackward(it EntryIterator) (EntryIterator, error) { - return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 128), forwardIter: it}, it.Error() + return &entryIteratorBackward{entries: make([]*logproto.Entry, 0, 128), forwardIter: it}, it.Error() +} + +// NewEntryIteratorForward returns an iterator which loads all the entries +// of an existing iterator, and then iterates over them backward. +func NewEntryIteratorForward(it EntryIterator) (EntryIterator, error) { + return &entryIteratorBackward{entries: make([]*logproto.Entry, 0, 128), forwardIter: it}, it.Error() } func (i *entryIteratorBackward) load() { @@ -461,13 +493,16 @@ func (i *entryIteratorBackward) Next() bool { return false } - i.cur = i.entries[len(i.entries)-1] - i.entries = i.entries[:len(i.entries)-1] + if i.forward { + i.cur, i.entries = i.entries[0], i.entries[1:] + } else { + i.cur, i.entries = i.entries[len(i.entries)-1], i.entries[:len(i.entries)-1] + } return true } -func (i *entryIteratorBackward) Entry() logproto.Entry { +func (i *entryIteratorBackward) Entry() *logproto.Entry { return i.cur } diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 531676456e6c..de52539eae5b 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -1,26 +1,28 @@ package logql import ( + "bytes" "fmt" "regexp" - "strings" "github.com/grafana/loki/pkg/iter" "github.com/prometheus/prometheus/pkg/labels" ) +type Filter func([]byte) bool + // QuerierFunc implements Querier. -type QuerierFunc func([]*labels.Matcher) (iter.EntryIterator, error) +type QuerierFunc func([]*labels.Matcher, Filter) (iter.EntryIterator, error) // Query implements Querier. -func (q QuerierFunc) Query(ms []*labels.Matcher) (iter.EntryIterator, error) { - return q(ms) +func (q QuerierFunc) Query(ms []*labels.Matcher, entryFilter Filter) (iter.EntryIterator, error) { + return q(ms, entryFilter) } // Querier allows a LogQL expression to fetch an EntryIterator for a // set of matchers. type Querier interface { - Query([]*labels.Matcher) (iter.EntryIterator, error) + Query([]*labels.Matcher, Filter) (iter.EntryIterator, error) } // Expr is a LogQL expression. @@ -34,7 +36,7 @@ type matchersExpr struct { } func (e *matchersExpr) Eval(q Querier) (iter.EntryIterator, error) { - return q.Query(e.matchers) + return q.Query(e.matchers, nil) } func (e *matchersExpr) Matchers() []*labels.Matcher { @@ -60,45 +62,52 @@ func NewFilterExpr(left Expr, ty labels.MatchType, match string) Expr { } } -func (e *filterExpr) Eval(q Querier) (iter.EntryIterator, error) { - var f func(string) bool +// todo recursion +func (e *filterExpr) filter() (func([]byte) bool, error) { + var f func([]byte) bool switch e.ty { case labels.MatchRegexp: re, err := regexp.Compile(e.match) if err != nil { return nil, err } - f = re.MatchString + f = re.Match case labels.MatchNotRegexp: re, err := regexp.Compile(e.match) if err != nil { return nil, err } - f = func(line string) bool { - return !re.MatchString(line) + f = func(line []byte) bool { + return !re.Match(line) } case labels.MatchEqual: - f = func(line string) bool { - return strings.Contains(line, e.match) + f = func(line []byte) bool { + return bytes.Contains(line, []byte(e.match)) } case labels.MatchNotEqual: - f = func(line string) bool { - return !strings.Contains(line, e.match) + f = func(line []byte) bool { + return bytes.Contains(line, []byte(e.match)) } default: return nil, fmt.Errorf("unknow matcher: %v", e.match) } + return f, nil +} - left, err := e.left.Eval(q) +func (e *filterExpr) Eval(q Querier) (iter.EntryIterator, error) { + f, err := e.filter() if err != nil { return nil, err } - - return iter.NewFilter(f, left), nil + next, err := q.Query(e.left.Matchers(), f) + if err != nil { + return nil, err + } + return next, nil } func mustNewMatcher(t labels.MatchType, n, v string) *labels.Matcher { diff --git a/pkg/storage/hack/main.go b/pkg/storage/hack/main.go index e09dc9ac7428..5e44345b8cef 100644 --- a/pkg/storage/hack/main.go +++ b/pkg/storage/hack/main.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "math/rand" + "os" "sync" "time" @@ -22,13 +23,17 @@ import ( "github.com/weaveworks/common/user" ) -var start = model.Time(1523750400000) - -var ctx = user.InjectOrgID(context.Background(), "fake") +var ( + start = model.Time(1523750400000) + ctx = user.InjectOrgID(context.Background(), "fake") + maxChunks = 600 // 600 chunks is 1.2bib of data enough to run benchmark +) +// fill up the local filesystem store with 1gib of data to run benchmark func main() { - // fill up the store with 1gib of data to run benchmark - fillStore() + if _, err := os.Stat("/tmp/benchmark/chunks"); os.IsNotExist(err) { + fillStore() + } } func getStore() (lstore.Store, error) { @@ -65,9 +70,9 @@ func fillStore() error { var wgPush sync.WaitGroup var flushCount int - // insert 5 streams with a random logs every seconds for an hour + // insert 5 streams with a random logs every nanoseconds // the string is randomize so chunks are big ~2mb - // take ~2min to build 1gib of data + // take ~1min to build 1gib of data for i := 0; i < 5; i++ { wgPush.Add(1) go func(j int) { @@ -80,7 +85,7 @@ func fillStore() error { labelsBuilder.Set(labels.MetricName, "logs") metric := labelsBuilder.Labels() fp := client.FastFingerprint(lbs) - chunkEnc := chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144) + chunkEnc := chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144, true) for ts := start.UnixNano(); ts < start.UnixNano()+time.Hour.Nanoseconds(); ts = ts + time.Millisecond.Nanoseconds() { entry := &logproto.Entry{ Timestamp: time.Unix(0, ts), @@ -100,11 +105,10 @@ func fillStore() error { } flushCount++ log.Println("flushed ", flushCount, from.UnixNano(), to.UnixNano(), metric) - if flushCount >= 600 { - // 600 chunk is enough data ~1gib + if flushCount >= maxChunks { return } - chunkEnc = chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144) + chunkEnc = chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144, true) } } diff --git a/pkg/storage/store.go b/pkg/storage/store.go index 8847a891706f..f89fe7d066e3 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -60,7 +60,7 @@ func (s *store) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter expr = logql.NewFilterExpr(expr, labels.MatchRegexp, req.Regex) } - querier := logql.QuerierFunc(func(matchers []*labels.Matcher) (iter.EntryIterator, error) { + querier := logql.QuerierFunc(func(matchers []*labels.Matcher, filter logql.Filter) (iter.EntryIterator, error) { nameLabelMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, "logs") if err != nil { return nil, err @@ -89,7 +89,7 @@ func (s *store) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter // we can proceed to filter the series that don't match. chksBySeries = filterSeriesByMatchers(chksBySeries, matchers) - iters, err := buildIterators(ctx, req, chksBySeries) + iters, err := buildIterators(ctx, req, chksBySeries, filter) if err != nil { return nil, err } @@ -125,21 +125,21 @@ outer: return chks } -func buildIterators(ctx context.Context, req *logproto.QueryRequest, chks map[model.Fingerprint][][]chunkenc.LazyChunk) ([]iter.EntryIterator, error) { +func buildIterators(ctx context.Context, req *logproto.QueryRequest, chks map[model.Fingerprint][][]chunkenc.LazyChunk, filter logql.Filter) ([]iter.EntryIterator, error) { result := make([]iter.EntryIterator, 0, len(chks)) for _, chunks := range chks { - iterator, err := buildHeapIterator(ctx, req, chunks) + iterator, err := buildHeapIterator(ctx, req, chunks, filter) if err != nil { return nil, err } - result = append(result, iterator) + result = append(result, iterator...) } return result, nil } -func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][]chunkenc.LazyChunk) (iter.EntryIterator, error) { +func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][]chunkenc.LazyChunk, filter logql.Filter) ([]iter.EntryIterator, error) { result := make([]iter.EntryIterator, 0, len(chks)) if chks[0][0].Chunk.Metric.Has("__name__") { labelsBuilder := labels.NewBuilder(chks[0][0].Chunk.Metric) @@ -151,17 +151,21 @@ func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][ for i := range chks { iterators := make([]iter.EntryIterator, 0, len(chks[i])) for j := range chks[i] { - iterator, err := chks[i][j].Iterator(ctx, req.Start, req.End, req.Direction) + iterator, err := chks[i][j].Iterator(ctx, req.Start, req.End, req.Direction, filter) if err != nil { return nil, err } iterators = append(iterators, iterator) } - + if req.Direction == logproto.FORWARD { + for i, j := 0, len(iterators)-1; i < j; i, j = i+1, j-1 { + iterators[i], iterators[j] = iterators[j], iterators[i] + } + } result = append(result, iter.NewNonOverlappingIterator(iterators, labels)) } - return iter.NewHeapIterator(result, req.Direction), nil + return result, nil } func loadFirstChunks(ctx context.Context, chks map[model.Fingerprint][][]chunkenc.LazyChunk) error { diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index c5534f3a952d..2df6418a5f41 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -4,9 +4,13 @@ import ( "context" "log" "runtime" + "runtime/debug" "testing" "time" + "net/http" + _ "net/http/pprof" + "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/local" "github.com/cortexproject/cortex/pkg/chunk/storage" @@ -17,36 +21,98 @@ import ( ) var ( - start = model.Time(1523750400000) - m runtime.MemStats - ctx = user.InjectOrgID(context.Background(), "fake") + start = model.Time(1523750400000) + m runtime.MemStats + ctx = user.InjectOrgID(context.Background(), "fake") + chunkStore = getStore() ) //go test -bench=. -benchmem -memprofile memprofile.out -cpuprofile profile.out -func Benchmark_store_LazyQuery(b *testing.B) { +func Benchmark_store_LazyQueryRegexBackward(b *testing.B) { + benchmarkStoreQuery(b, &logproto.QueryRequest{ + Query: "{foo=\"bar\"}", + Regex: "fuzz", + Limit: 1000, + Start: time.Unix(0, start.UnixNano()), + End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), + Direction: logproto.BACKWARD, + }) +} - for i := 0; i < b.N; i++ { - store, err := getStore() - if err != nil { - b.Fatal(err) +func Benchmark_store_LazyQueryLogQLBackward(b *testing.B) { + benchmarkStoreQuery(b, &logproto.QueryRequest{ + Query: "{foo=\"bar\"} |= \"test\" != \"toto\"", + Regex: "fuzz", + Limit: 1000, + Start: time.Unix(0, start.UnixNano()), + End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), + Direction: logproto.BACKWARD, + }) +} + +func Benchmark_store_LazyQueryRegexForward(b *testing.B) { + benchmarkStoreQuery(b, &logproto.QueryRequest{ + Query: "{foo=\"bar\"}", + Regex: "fuzz", + Limit: 1000, + Start: time.Unix(0, start.UnixNano()), + End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), + Direction: logproto.FORWARD, + }) +} + +func Benchmark_store_LazyQueryForward(b *testing.B) { + benchmarkStoreQuery(b, &logproto.QueryRequest{ + Query: "{foo=\"bar\"}", + Limit: 1000, + Start: time.Unix(0, start.UnixNano()), + End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), + Direction: logproto.FORWARD, + }) +} + +func Benchmark_store_LazyQueryBackward(b *testing.B) { + benchmarkStoreQuery(b, &logproto.QueryRequest{ + Query: "{foo=\"bar\"}", + Limit: 1000, + Start: time.Unix(0, start.UnixNano()), + End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), + Direction: logproto.BACKWARD, + }) +} + +func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { + b.ReportAllocs() + // force to run gc 10x more often + debug.SetGCPercent(10) + stop := make(chan struct{}) + go func() { + http.ListenAndServe(":6060", http.DefaultServeMux) + }() + go func() { + ticker := time.NewTicker(time.Millisecond) + for { + select { + case <-ticker.C: + // print and capture the max in use heap size + printHeap(b, false) + case <-stop: + ticker.Stop() + return + } } - iter, err := store.LazyQuery(ctx, &logproto.QueryRequest{ - Query: "{foo=\"bar\"}", - Regex: "fuzz", - Limit: 1000, - Start: time.Unix(0, start.UnixNano()), - End: time.Unix(0, (24*time.Hour.Nanoseconds())+start.UnixNano()), - Direction: logproto.BACKWARD, - }) + }() + for i := 0; i < b.N; i++ { + iter, err := chunkStore.LazyQuery(ctx, query) if err != nil { b.Fatal(err) } - res := []logproto.Entry{} - printHeap(b) + res := []*logproto.Entry{} + printHeap(b, true) j := 0 for iter.Next() { j++ - printHeap(b) + printHeap(b, false) res = append(res, iter.Entry()) // todo this should be done in the store. if j == 1000 { @@ -54,18 +120,25 @@ func Benchmark_store_LazyQuery(b *testing.B) { } } iter.Close() - printHeap(b) + printHeap(b, true) log.Println("line fetched", len(res)) - store.Stop() } + close(stop) } -func printHeap(b *testing.B) { +var maxHeapInuse uint64 + +func printHeap(b *testing.B, show bool) { runtime.ReadMemStats(&m) - log.Printf("Benchmark %d HeapInuse: %d Mbytes\n", b.N, m.HeapInuse/1024/1024) + if m.HeapInuse > maxHeapInuse { + maxHeapInuse = m.HeapInuse + } + if show { + log.Printf("Benchmark %d maxHeapInuse: %d Mbytes\n", b.N, maxHeapInuse/1024/1024) + } } -func getStore() (Store, error) { +func getStore() Store { store, err := NewStore(storage.Config{ BoltDBConfig: local.BoltDBConfig{Directory: "/tmp/benchmark/index"}, FSConfig: local.FSConfig{Directory: "/tmp/benchmark/chunks"}, @@ -84,7 +157,7 @@ func getStore() (Store, error) { }, }, &validation.Overrides{}) if err != nil { - return nil, err + panic(err) } - return store, nil + return store } diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 80f4ce43526c..c0730abce92f 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -21,7 +21,8 @@ tracing: config: auth_enabled: false ingester: - chunk_idle_period: 15m + chunk_idle_period: 5m + chunk_retain_period: 30s chunk_block_size: 262144 lifecycler: ring: From e5d06bba410523063ecaa75dbdc7625ad15c8306 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Sun, 30 Jun 2019 11:51:48 -0400 Subject: [PATCH 08/25] revert pointer on logentry --- pkg/chunkenc/dumb_chunk.go | 8 ++++---- pkg/chunkenc/gzip.go | 8 ++++---- pkg/iter/iterator.go | 30 +++++++++++++++--------------- pkg/storage/store_test.go | 2 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/chunkenc/dumb_chunk.go b/pkg/chunkenc/dumb_chunk.go index 9c31ca0a9a9e..19446634f7cf 100644 --- a/pkg/chunkenc/dumb_chunk.go +++ b/pkg/chunkenc/dumb_chunk.go @@ -19,7 +19,7 @@ func NewDumbChunk() Chunk { } type dumbChunk struct { - entries []*logproto.Entry + entries []logproto.Entry } func (c *dumbChunk) Bounds() (time.Time, time.Time) { @@ -42,7 +42,7 @@ func (c *dumbChunk) Append(entry *logproto.Entry) error { return ErrOutOfOrder } - c.entries = append(c.entries, entry) + c.entries = append(c.entries, *entry) return nil } @@ -84,7 +84,7 @@ func (c *dumbChunk) Bytes() ([]byte, error) { type dumbChunkIterator struct { direction logproto.Direction i int - entries []*logproto.Entry + entries []logproto.Entry } func (i *dumbChunkIterator) Next() bool { @@ -100,7 +100,7 @@ func (i *dumbChunkIterator) Next() bool { } } -func (i *dumbChunkIterator) Entry() *logproto.Entry { +func (i *dumbChunkIterator) Entry() logproto.Entry { return i.entries[i.i] } diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 14c3924cf2af..2673c289a63e 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -465,8 +465,8 @@ func (li *listIterator) Next() bool { return false } -func (li *listIterator) Entry() *logproto.Entry { - return &logproto.Entry{ +func (li *listIterator) Entry() logproto.Entry { + return logproto.Entry{ Timestamp: time.Unix(0, li.cur.t), Line: li.cur.s, } @@ -554,9 +554,9 @@ func (si *bufferedIterator) Next() bool { return true } -func (si *bufferedIterator) Entry() *logproto.Entry { +func (si *bufferedIterator) Entry() logproto.Entry { // log.Println("lineParsed", atomic.AddInt64(&lineParsed, 1)) - return &si.cur + return si.cur } func (si *bufferedIterator) Error() error { return si.err } diff --git a/pkg/iter/iterator.go b/pkg/iter/iterator.go index 309fa98487dd..e93d631bcbef 100644 --- a/pkg/iter/iterator.go +++ b/pkg/iter/iterator.go @@ -14,7 +14,7 @@ import ( // EntryIterator iterates over entries in time-order. type EntryIterator interface { Next() bool - Entry() *logproto.Entry + Entry() logproto.Entry Labels() string Error() error Close() error @@ -49,8 +49,8 @@ func (i *streamIterator) Labels() string { return i.labels } -func (i *streamIterator) Entry() *logproto.Entry { - return &i.entries[i.i] +func (i *streamIterator) Entry() logproto.Entry { + return i.entries[i.i] } func (i *streamIterator) Close() error { @@ -117,7 +117,7 @@ type heapIterator struct { prenext bool tuples tuples - currEntry *logproto.Entry + currEntry logproto.Entry currLabels string errs []error } @@ -156,7 +156,7 @@ func (i *heapIterator) Push(ei EntryIterator) { } type tuple struct { - *logproto.Entry + logproto.Entry EntryIterator } @@ -233,7 +233,7 @@ func mostCommon(tuples tuples) tuple { return result } -func (i *heapIterator) Entry() *logproto.Entry { +func (i *heapIterator) Entry() logproto.Entry { return i.currEntry } @@ -310,7 +310,7 @@ func (i *queryClientIterator) Next() bool { return true } -func (i *queryClientIterator) Entry() *logproto.Entry { +func (i *queryClientIterator) Entry() logproto.Entry { return i.curr.Entry() } @@ -330,7 +330,7 @@ type filter struct { EntryIterator f func(string) bool - curr *logproto.Entry + curr logproto.Entry } // NewFilter builds a filtering iterator. @@ -353,7 +353,7 @@ func (i *filter) Next() bool { return false } -func (i *filter) Entry() *logproto.Entry { +func (i *filter) Entry() logproto.Entry { return i.curr } @@ -392,7 +392,7 @@ func (i *nonOverlappingIterator) Next() bool { return true } -func (i *nonOverlappingIterator) Entry() *logproto.Entry { +func (i *nonOverlappingIterator) Entry() logproto.Entry { return i.curr.Entry() } @@ -457,8 +457,8 @@ func (i *timeRangedIterator) Next() bool { type entryIteratorBackward struct { forwardIter EntryIterator - cur *logproto.Entry - entries []*logproto.Entry + cur logproto.Entry + entries []logproto.Entry loaded bool forward bool @@ -467,13 +467,13 @@ type entryIteratorBackward struct { // NewEntryIteratorBackward returns an iterator which loads all the entries // of an existing iterator, and then iterates over them backward. func NewEntryIteratorBackward(it EntryIterator) (EntryIterator, error) { - return &entryIteratorBackward{entries: make([]*logproto.Entry, 0, 128), forwardIter: it}, it.Error() + return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 128), forwardIter: it}, it.Error() } // NewEntryIteratorForward returns an iterator which loads all the entries // of an existing iterator, and then iterates over them backward. func NewEntryIteratorForward(it EntryIterator) (EntryIterator, error) { - return &entryIteratorBackward{entries: make([]*logproto.Entry, 0, 128), forwardIter: it}, it.Error() + return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 128), forwardIter: it}, it.Error() } func (i *entryIteratorBackward) load() { @@ -502,7 +502,7 @@ func (i *entryIteratorBackward) Next() bool { return true } -func (i *entryIteratorBackward) Entry() *logproto.Entry { +func (i *entryIteratorBackward) Entry() logproto.Entry { return i.cur } diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index 2df6418a5f41..35dfa52deb5b 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -107,7 +107,7 @@ func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { if err != nil { b.Fatal(err) } - res := []*logproto.Entry{} + res := []logproto.Entry{} printHeap(b, true) j := 0 for iter.Next() { From 2f683d575119402d76a120d74d347f1564df3478 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Sun, 30 Jun 2019 19:12:40 -0400 Subject: [PATCH 09/25] fix ingester to use filter --- Makefile | 4 +--- pkg/chunkenc/facade.go | 2 +- pkg/chunkenc/gzip.go | 24 ++++++++++++------------ pkg/ingester/chunk_test.go | 4 ++-- pkg/ingester/flush_test.go | 2 +- pkg/ingester/instance.go | 8 ++++---- pkg/ingester/stream.go | 5 +++-- pkg/ingester/stream_test.go | 4 ++-- pkg/ingester/tailer.go | 19 ++++++++++--------- pkg/logql/ast.go | 11 ++++++++++- pkg/storage/hack/main.go | 4 ++-- 11 files changed, 48 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 09ae801c648f..de42793b655a 100644 --- a/Makefile +++ b/Makefile @@ -313,8 +313,6 @@ push-plugin: build-plugin enable-plugin: docker plugin enable grafana/loki-docker-driver:$(PLUGIN_TAG) -fill-store: +benchmark-store: go run ./pkg/storage/hack/main.go - -benchmark-store: fill-store go test ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out \ No newline at end of file diff --git a/pkg/chunkenc/facade.go b/pkg/chunkenc/facade.go index eb8820ff1b51..ef3801d1dc22 100644 --- a/pkg/chunkenc/facade.go +++ b/pkg/chunkenc/facade.go @@ -12,7 +12,7 @@ const GzipLogChunk = encoding.Encoding(128) func init() { encoding.MustRegisterEncoding(GzipLogChunk, "GzipLogChunk", func() encoding.Chunk { return &Facade{ - c: NewMemChunk(EncGZIP, false), + c: NewMemChunk(EncGZIP), } }) } diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 2673c289a63e..46ec6e360458 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -129,7 +129,7 @@ type entry struct { // NewMemChunkSize returns a new in-mem chunk. // Mainly for config push size. -func NewMemChunkSize(enc Encoding, blockSize int, head bool) *MemChunk { +func NewMemChunkSize(enc Encoding, blockSize int) *MemChunk { c := &MemChunk{ blockSize: blockSize, // The blockSize in bytes. blocks: []block{}, @@ -139,9 +139,7 @@ func NewMemChunkSize(enc Encoding, blockSize int, head bool) *MemChunk { encoding: enc, } - if head { - c.head = &headBlock{} - } + c.head = &headBlock{} switch enc { case EncGZIP: @@ -154,8 +152,8 @@ func NewMemChunkSize(enc Encoding, blockSize int, head bool) *MemChunk { } // NewMemChunk returns a new in-mem chunk for query. -func NewMemChunk(enc Encoding, head bool) *MemChunk { - return NewMemChunkSize(enc, 256*1024, head) +func NewMemChunk(enc Encoding) *MemChunk { + return NewMemChunkSize(enc, 256*1024) } // NewByteChunk returns a MemChunk on the passed bytes. @@ -398,8 +396,7 @@ func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction } if c.head != nil { - // todo filter - its = append(its, c.head.iterator(mint, maxt)) + its = append(its, c.head.iterator(mint, maxt, filter)) } if direction == logproto.FORWARD { @@ -428,7 +425,7 @@ func (b block) iterator(pool CompressionPool, filter logql.Filter) (iter.EntryIt return newBufferedIterator(pool, b.b, filter), nil } -func (hb *headBlock) iterator(mint, maxt int64) iter.EntryIterator { +func (hb *headBlock) iterator(mint, maxt int64, filter logql.Filter) iter.EntryIterator { if hb.isEmpty() || (maxt < hb.mint || hb.maxt < mint) { return emptyIterator } @@ -438,8 +435,12 @@ func (hb *headBlock) iterator(mint, maxt int64) iter.EntryIterator { // but the tradeoff is that queries to near-realtime data would be much lower than // cutting of blocks. - entries := make([]entry, len(hb.entries)) - copy(entries, hb.entries) + entries := make([]entry, 0, len(hb.entries)) + for _, e := range entries { + if filter == nil || filter([]byte(e.s)) { + entries = append(entries, e) + } + } return &listIterator{ entries: entries, @@ -555,7 +556,6 @@ func (si *bufferedIterator) Next() bool { } func (si *bufferedIterator) Entry() logproto.Entry { - // log.Println("lineParsed", atomic.AddInt64(&lineParsed, 1)) return si.cur } diff --git a/pkg/ingester/chunk_test.go b/pkg/ingester/chunk_test.go index 4feb7fffe778..e7ab04b6ed9d 100644 --- a/pkg/ingester/chunk_test.go +++ b/pkg/ingester/chunk_test.go @@ -60,7 +60,7 @@ func TestIterator(t *testing.T) { for i := 0; i < entries; i++ { from := rand.Intn(entries - 1) len := rand.Intn(entries-from) + 1 - iter, err := chunk.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.FORWARD) + iter, err := chunk.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.FORWARD, nil) require.NoError(t, err) testIteratorForward(t, iter, int64(from), int64(from+len)) _ = iter.Close() @@ -69,7 +69,7 @@ func TestIterator(t *testing.T) { for i := 0; i < entries; i++ { from := rand.Intn(entries - 1) len := rand.Intn(entries-from) + 1 - iter, err := chunk.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.BACKWARD) + iter, err := chunk.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.BACKWARD, nil) require.NoError(t, err) testIteratorBackward(t, iter, int64(from), int64(from+len)) _ = iter.Close() diff --git a/pkg/ingester/flush_test.go b/pkg/ingester/flush_test.go index 4b2f850db208..aab3f96f3862 100644 --- a/pkg/ingester/flush_test.go +++ b/pkg/ingester/flush_test.go @@ -183,7 +183,7 @@ func (s *testStore) checkData(t *testing.T, userIDs []string, testData map[strin } func buildStreamsFromChunk(t *testing.T, labels string, chk chunkenc.Chunk) *logproto.Stream { - it, err := chk.Iterator(time.Unix(0, 0), time.Unix(1000, 0), logproto.FORWARD) + it, err := chk.Iterator(time.Unix(0, 0), time.Unix(1000, 0), logproto.FORWARD, nil) require.NoError(t, err) stream := &logproto.Stream{ diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 3d79fd6bb92f..d2e66fdd01a6 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -115,8 +115,8 @@ func (i *instance) Query(req *logproto.QueryRequest) (iter.EntryIterator, error) expr = logql.NewFilterExpr(expr, labels.MatchRegexp, req.Regex) } - querier := logql.QuerierFunc(func(matchers []*labels.Matcher) (iter.EntryIterator, error) { - iters, err := i.lookupStreams(req, matchers) + querier := logql.QuerierFunc(func(matchers []*labels.Matcher, filter logql.Filter) (iter.EntryIterator, error) { + iters, err := i.lookupStreams(req, matchers, filter) if err != nil { return nil, err } @@ -151,7 +151,7 @@ func (i *instance) Label(_ context.Context, req *logproto.LabelRequest) (*logpro }, nil } -func (i *instance) lookupStreams(req *logproto.QueryRequest, matchers []*labels.Matcher) ([]iter.EntryIterator, error) { +func (i *instance) lookupStreams(req *logproto.QueryRequest, matchers []*labels.Matcher, filter logql.Filter) ([]iter.EntryIterator, error) { i.streamsMtx.RLock() defer i.streamsMtx.RUnlock() @@ -171,7 +171,7 @@ outer: continue outer } } - iter, err := stream.Iterator(req.Start, req.End, req.Direction) + iter, err := stream.Iterator(req.Start, req.End, req.Direction, filter) if err != nil { return nil, err } diff --git a/pkg/ingester/stream.go b/pkg/ingester/stream.go index ccd7b6e6c083..f3aab3d79760 100644 --- a/pkg/ingester/stream.go +++ b/pkg/ingester/stream.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/loki/pkg/chunkenc" "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql" ) var ( @@ -142,10 +143,10 @@ func (s *stream) Push(_ context.Context, entries []logproto.Entry) error { } // Returns an iterator. -func (s *stream) Iterator(from, through time.Time, direction logproto.Direction) (iter.EntryIterator, error) { +func (s *stream) Iterator(from, through time.Time, direction logproto.Direction, filter logql.Filter) (iter.EntryIterator, error) { iterators := make([]iter.EntryIterator, 0, len(s.chunks)) for _, c := range s.chunks { - itr, err := c.chunk.Iterator(from, through, direction) + itr, err := c.chunk.Iterator(from, through, direction, filter) if err != nil { return nil, err } diff --git a/pkg/ingester/stream_test.go b/pkg/ingester/stream_test.go index a61b100b16c5..3febb271eada 100644 --- a/pkg/ingester/stream_test.go +++ b/pkg/ingester/stream_test.go @@ -40,7 +40,7 @@ func TestStreamIterator(t *testing.T) { for i := 0; i < 100; i++ { from := rand.Intn(chunks*entries - 1) len := rand.Intn(chunks*entries-from) + 1 - iter, err := s.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.FORWARD) + iter, err := s.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.FORWARD, nil) require.NotNil(t, iter) require.NoError(t, err) testIteratorForward(t, iter, int64(from), int64(from+len)) @@ -50,7 +50,7 @@ func TestStreamIterator(t *testing.T) { for i := 0; i < 100; i++ { from := rand.Intn(entries - 1) len := rand.Intn(chunks*entries-from) + 1 - iter, err := s.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.BACKWARD) + iter, err := s.Iterator(time.Unix(int64(from), 0), time.Unix(int64(from+len), 0), logproto.BACKWARD, nil) require.NotNil(t, iter) require.NoError(t, err) testIteratorBackward(t, iter, int64(from), int64(from+len)) diff --git a/pkg/ingester/tailer.go b/pkg/ingester/tailer.go index c37ced48f14e..c332b2765b8a 100644 --- a/pkg/ingester/tailer.go +++ b/pkg/ingester/tailer.go @@ -129,21 +129,22 @@ func (t *tailer) send(stream logproto.Stream) { } func (t *tailer) filterEntriesInStream(stream *logproto.Stream) error { - querier := logql.QuerierFunc(func(matchers []*labels.Matcher) (iter.EntryIterator, error) { - return iter.NewStreamIterator(stream), nil + querier := logql.QuerierFunc(func(matchers []*labels.Matcher, filter logql.Filter) (iter.EntryIterator, error) { + var filteredEntries []logproto.Entry + for _, e := range stream.Entries { + if filter == nil || filter([]byte(e.Line)) { + filteredEntries = append(filteredEntries, e) + } + } + stream.Entries = filteredEntries + return nil, nil }) - itr, err := t.expr.Eval(querier) + _, err := t.expr.Eval(querier) if err != nil { return err } - filteredEntries := new([]logproto.Entry) - for itr.Next() { - *filteredEntries = append(*filteredEntries, itr.Entry()) - } - - stream.Entries = *filteredEntries return nil } diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index de52539eae5b..1ba74c13fc6c 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -62,7 +62,6 @@ func NewFilterExpr(left Expr, ty labels.MatchType, match string) Expr { } } -// todo recursion func (e *filterExpr) filter() (func([]byte) bool, error) { var f func([]byte) bool switch e.ty { @@ -95,6 +94,16 @@ func (e *filterExpr) filter() (func([]byte) bool, error) { default: return nil, fmt.Errorf("unknow matcher: %v", e.match) } + next, ok := e.left.(*filterExpr) + if ok { + nextFilter, err := next.filter() + if err != nil { + return nil, err + } + return func(line []byte) bool { + return nextFilter(line) && f(line) + }, nil + } return f, nil } diff --git a/pkg/storage/hack/main.go b/pkg/storage/hack/main.go index 5e44345b8cef..bc8446941151 100644 --- a/pkg/storage/hack/main.go +++ b/pkg/storage/hack/main.go @@ -85,7 +85,7 @@ func fillStore() error { labelsBuilder.Set(labels.MetricName, "logs") metric := labelsBuilder.Labels() fp := client.FastFingerprint(lbs) - chunkEnc := chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144, true) + chunkEnc := chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144) for ts := start.UnixNano(); ts < start.UnixNano()+time.Hour.Nanoseconds(); ts = ts + time.Millisecond.Nanoseconds() { entry := &logproto.Entry{ Timestamp: time.Unix(0, ts), @@ -108,7 +108,7 @@ func fillStore() error { if flushCount >= maxChunks { return } - chunkEnc = chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144, true) + chunkEnc = chunkenc.NewMemChunkSize(chunkenc.EncGZIP, 262144) } } From 5eb0b63d6c4a130a6603c3dea8d07fc991e1f2c5 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 2 Jul 2019 09:38:35 -0400 Subject: [PATCH 10/25] fix list iterator --- pkg/chunkenc/gzip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 46ec6e360458..b05401144bc7 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -436,7 +436,7 @@ func (hb *headBlock) iterator(mint, maxt int64, filter logql.Filter) iter.EntryI // cutting of blocks. entries := make([]entry, 0, len(hb.entries)) - for _, e := range entries { + for _, e := range hb.entries { if filter == nil || filter([]byte(e.s)) { entries = append(entries, e) } From d3ae9f0de44b9bd591fdf2fe140c8902f122f3ad Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 3 Jul 2019 11:27:51 -0400 Subject: [PATCH 11/25] fixes linter --- pkg/chunkenc/gzip.go | 79 +++++++++---------- pkg/chunkenc/gzip_test.go | 23 +++--- pkg/chunkenc/interface.go | 1 - pkg/chunkenc/lazy_chunk.go | 4 - pkg/iter/iterator.go | 56 +++---------- pkg/logql/ast.go | 1 + pkg/storage/hack/main.go | 39 +++++---- pkg/storage/store.go | 1 + pkg/storage/store_test.go | 11 ++- pkg/util/conv.go | 25 +++++- .../helm/loki/templates/statefulset.yaml | 2 + production/helm/loki/values.yaml | 4 +- 12 files changed, 112 insertions(+), 134 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index b05401144bc7..bf7caa50713b 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -134,13 +134,11 @@ func NewMemChunkSize(enc Encoding, blockSize int) *MemChunk { blockSize: blockSize, // The blockSize in bytes. blocks: []block{}, - head: nil, + head: &headBlock{}, encoding: enc, } - c.head = &headBlock{} - switch enc { case EncGZIP: c.cPool = &Gzip @@ -160,7 +158,7 @@ func NewMemChunk(enc Encoding) *MemChunk { func NewByteChunk(b []byte) (*MemChunk, error) { bc := &MemChunk{ cPool: &Gzip, - head: nil, // Dummy, empty headblock. + head: &headBlock{}, // Dummy, empty headblock. } db := decbuf{b: b} @@ -303,7 +301,7 @@ func (c *MemChunk) Size() int { ne += blk.numEntries } - if c.head != nil && !c.head.isEmpty() { + if !c.head.isEmpty() { ne += len(c.head.entries) } @@ -367,7 +365,7 @@ func (c *MemChunk) Bounds() (fromT, toT time.Time) { to = c.blocks[len(c.blocks)-1].maxt } - if c.head != nil && !c.head.isEmpty() { + if !c.head.isEmpty() { if from == 0 || from > c.head.mint { from = c.head.mint } @@ -387,24 +385,14 @@ func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction for _, b := range c.blocks { if maxt > b.mint && b.maxt > mint { - it, err := b.iterator(c.cPool, filter) - if err != nil { - return nil, err - } - its = append(its, it) + its = append(its, b.iterator(c.cPool, filter)) } } - if c.head != nil { + if !c.head.isEmpty() { its = append(its, c.head.iterator(mint, maxt, filter)) } - if direction == logproto.FORWARD { - for i, j := 0, len(its)-1; i < j; i, j = i+1, j-1 { - its[i], its[j] = its[j], its[i] - } - } - iterForward := iter.NewTimeRangedIterator( iter.NewNonOverlappingIterator(its, ""), time.Unix(0, mint), @@ -418,11 +406,11 @@ func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction return iter.NewEntryIteratorBackward(iterForward) } -func (b block) iterator(pool CompressionPool, filter logql.Filter) (iter.EntryIterator, error) { +func (b block) iterator(pool CompressionPool, filter logql.Filter) iter.EntryIterator { if len(b.b) == 0 { - return emptyIterator, nil + return emptyIterator } - return newBufferedIterator(pool, b.b, filter), nil + return newBufferedIterator(pool, b.b, filter) } func (hb *headBlock) iterator(mint, maxt int64, filter logql.Filter) iter.EntryIterator { @@ -483,7 +471,6 @@ type bufferedIterator struct { pool CompressionPool cur logproto.Entry - l uint64 err error @@ -495,8 +482,6 @@ type bufferedIterator struct { filter logql.Filter } -var lineParsed int64 - func newBufferedIterator(pool CompressionPool, b []byte, filter logql.Filter) *bufferedIterator { r := pool.GetReader(bytes.NewBuffer(b)) return &bufferedIterator{ @@ -510,49 +495,57 @@ func newBufferedIterator(pool CompressionPool, b []byte, filter logql.Filter) *b } func (si *bufferedIterator) Next() bool { + for { + ts, line, ok := si.moveNext() + if !ok { + si.Close() + return false + } + if si.filter != nil && !si.filter(line) { + continue + } + si.cur.Line = string(line) + si.cur.Timestamp = time.Unix(0, ts) + return true + } +} + +// moveNext moves the buffer to the next entry +func (si *bufferedIterator) moveNext() (int64, []byte, bool) { ts, err := binary.ReadVarint(si.s) if err != nil { if err != io.EOF { si.err = err } - si.Close() - return false + return 0, nil, false } - si.l, err = binary.ReadUvarint(si.s) + l, err := binary.ReadUvarint(si.s) if err != nil { if err != io.EOF { si.err = err - si.Close() - return false + return 0, nil, false } } - for len(si.buf) < int(si.l) { + for len(si.buf) < int(l) { si.buf = append(si.buf, make([]byte, 256)...) } - n, err := si.s.Read(si.buf[:si.l]) + n, err := si.s.Read(si.buf[:l]) if err != nil && err != io.EOF { si.err = err - si.Close() - return false + return 0, nil, false } - if n < int(si.l) { - _, err = si.s.Read(si.buf[n:si.l]) + if n < int(l) { + _, err = si.s.Read(si.buf[n:l]) if err != nil { si.err = err - si.Close() - return false + return 0, nil, false } } - if si.filter != nil && !si.filter(si.buf[:si.l]) { - return si.Next() - } - si.cur.Line = string(si.buf[:si.l]) - si.cur.Timestamp = time.Unix(0, ts) - return true + return ts, si.buf[:l], true } func (si *bufferedIterator) Entry() logproto.Entry { diff --git a/pkg/chunkenc/gzip_test.go b/pkg/chunkenc/gzip_test.go index d4c363e6d30c..bb52477c5e83 100644 --- a/pkg/chunkenc/gzip_test.go +++ b/pkg/chunkenc/gzip_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/grafana/loki/pkg/logproto" - "github.com/stretchr/testify/require" ) @@ -77,7 +76,7 @@ func TestGZIPBlock(t *testing.T) { } } - it, err := chk.Iterator(time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD) + it, err := chk.Iterator(time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD, nil) require.NoError(t, err) idx := 0 @@ -92,7 +91,7 @@ func TestGZIPBlock(t *testing.T) { require.Equal(t, len(cases), idx) t.Run("bounded-iteration", func(t *testing.T) { - it, err := chk.Iterator(time.Unix(0, 3), time.Unix(0, 7), logproto.FORWARD) + it, err := chk.Iterator(time.Unix(0, 3), time.Unix(0, 7), logproto.FORWARD, nil) require.NoError(t, err) idx := 2 @@ -134,7 +133,7 @@ func TestGZIPCompression(t *testing.T) { require.NoError(t, err) fmt.Println(float64(len(b))/(1024*1024), float64(len(b2))/(1024*1024), float64(len(b2))/float64(len(chk.blocks))) - it, err := chk.Iterator(time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD) + it, err := chk.Iterator(time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD, nil) require.NoError(t, err) for i, l := range lines { @@ -164,7 +163,7 @@ func TestGZIPSerialisation(t *testing.T) { bc, err := NewByteChunk(byt) require.NoError(t, err) - it, err := bc.Iterator(time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD) + it, err := bc.Iterator(time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD, nil) require.NoError(t, err) for i := 0; i < numSamples; i++ { require.True(t, it.Next()) @@ -205,7 +204,7 @@ func TestGZIPChunkFilling(t *testing.T) { require.Equal(t, int64(lines), i) - it, err := chk.Iterator(time.Unix(0, 0), time.Unix(0, 100), logproto.FORWARD) + it, err := chk.Iterator(time.Unix(0, 0), time.Unix(0, 100), logproto.FORWARD, nil) require.NoError(t, err) i = 0 for it.Next() { @@ -229,10 +228,10 @@ func BenchmarkWriteGZIP(b *testing.B) { i := int64(0) for n := 0; n < b.N; n++ { - c := NewMemChunk(EncGZIP, true) + c := NewMemChunk(EncGZIP) // adds until full so we trigger cut which serialize using gzip for c.SpaceFor(entry) { - c.Append(entry) + _ = c.Append(entry) entry.Timestamp = time.Unix(0, i) i++ } @@ -251,10 +250,10 @@ func BenchmarkReadGZIP(b *testing.B) { i := int64(0) for n := 0; n < 50; n++ { - c := NewMemChunk(EncGZIP, true) + c := NewMemChunk(EncGZIP) // adds until full so we trigger cut which serialize using gzip for c.SpaceFor(entry) { - c.Append(entry) + _ = c.Append(entry) entry.Timestamp = time.Unix(0, i) i++ } @@ -266,9 +265,9 @@ func BenchmarkReadGZIP(b *testing.B) { for _, c := range chunks { wg.Add(1) go func(c Chunk) { - iterator, err := c.Iterator(time.Unix(0, 0), time.Unix(0, 10), logproto.BACKWARD) + iterator, err := c.Iterator(time.Unix(0, 0), time.Unix(0, 10), logproto.BACKWARD, nil) if err != nil { - b.Fatal(err) + panic(err) } for iterator.Next() { entries = append(entries, iterator.Entry()) diff --git a/pkg/chunkenc/interface.go b/pkg/chunkenc/interface.go index 3b49b5e2c4e3..08ee8cf3034c 100644 --- a/pkg/chunkenc/interface.go +++ b/pkg/chunkenc/interface.go @@ -64,5 +64,4 @@ type CompressionWriter interface { type CompressionReader interface { Read(p []byte) (int, error) Reset(r io.Reader) error - Close() error } diff --git a/pkg/chunkenc/lazy_chunk.go b/pkg/chunkenc/lazy_chunk.go index 299b937da171..9135f69fb3e9 100644 --- a/pkg/chunkenc/lazy_chunk.go +++ b/pkg/chunkenc/lazy_chunk.go @@ -59,9 +59,6 @@ type lazyIterator struct { closed bool } -var chunksOpen int64 -var chunksTotal int64 - func (it *lazyIterator) Next() bool { if it.err != nil { return false @@ -105,7 +102,6 @@ func (it *lazyIterator) Error() error { func (it *lazyIterator) Close() error { if it.EntryIterator != nil { - // log.Println("Chunk Open", atomic.AddInt64(&chunksOpen, -1)) it.chunk = nil it.closed = true err := it.EntryIterator.Close() diff --git a/pkg/iter/iterator.go b/pkg/iter/iterator.go index e93d631bcbef..4c2f33a0c9b8 100644 --- a/pkg/iter/iterator.go +++ b/pkg/iter/iterator.go @@ -326,42 +326,13 @@ func (i *queryClientIterator) Close() error { return i.client.CloseSend() } -type filter struct { - EntryIterator - f func(string) bool - - curr logproto.Entry -} - -// NewFilter builds a filtering iterator. -func NewFilter(f func(string) bool, i EntryIterator) EntryIterator { - return &filter{ - f: f, - EntryIterator: i, - } -} - -func (i *filter) Next() bool { - for i.EntryIterator.Next() { - curr := i.EntryIterator.Entry() - if i.f(curr.Line) { - i.curr = curr - return true - } - } - i.EntryIterator.Close() - return false -} - -func (i *filter) Entry() logproto.Entry { - return i.curr -} - type nonOverlappingIterator struct { labels string i int iterators []EntryIterator curr EntryIterator + + lastEntry *logproto.Entry } // NewNonOverlappingIterator gives a chained iterator over a list of iterators. @@ -393,7 +364,12 @@ func (i *nonOverlappingIterator) Next() bool { } func (i *nonOverlappingIterator) Entry() logproto.Entry { - return i.curr.Entry() + if i.curr == nil { + return *i.lastEntry + } + entry := i.curr.Entry() + i.lastEntry = &entry + return *i.lastEntry } func (i *nonOverlappingIterator) Labels() string { @@ -460,8 +436,6 @@ type entryIteratorBackward struct { cur logproto.Entry entries []logproto.Entry loaded bool - - forward bool } // NewEntryIteratorBackward returns an iterator which loads all the entries @@ -470,12 +444,6 @@ func NewEntryIteratorBackward(it EntryIterator) (EntryIterator, error) { return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 128), forwardIter: it}, it.Error() } -// NewEntryIteratorForward returns an iterator which loads all the entries -// of an existing iterator, and then iterates over them backward. -func NewEntryIteratorForward(it EntryIterator) (EntryIterator, error) { - return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 128), forwardIter: it}, it.Error() -} - func (i *entryIteratorBackward) load() { if !i.loaded { i.loaded = true @@ -492,13 +460,7 @@ func (i *entryIteratorBackward) Next() bool { i.entries = nil return false } - - if i.forward { - i.cur, i.entries = i.entries[0], i.entries[1:] - } else { - i.cur, i.entries = i.entries[len(i.entries)-1], i.entries[:len(i.entries)-1] - } - + i.cur, i.entries = i.entries[len(i.entries)-1], i.entries[:len(i.entries)-1] return true } diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 1ba74c13fc6c..eaa683459882 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" ) +// Filter is a line filter sent to a querier to filter out log line. type Filter func([]byte) bool // QuerierFunc implements Querier. diff --git a/pkg/storage/hack/main.go b/pkg/storage/hack/main.go index bc8446941151..c52f573a75a1 100644 --- a/pkg/storage/hack/main.go +++ b/pkg/storage/hack/main.go @@ -32,28 +32,35 @@ var ( // fill up the local filesystem store with 1gib of data to run benchmark func main() { if _, err := os.Stat("/tmp/benchmark/chunks"); os.IsNotExist(err) { - fillStore() + if err := fillStore(); err != nil { + log.Fatal("error filling up storage:", err) + } } } func getStore() (lstore.Store, error) { - store, err := lstore.NewStore(storage.Config{ - BoltDBConfig: local.BoltDBConfig{Directory: "/tmp/benchmark/index"}, - FSConfig: local.FSConfig{Directory: "/tmp/benchmark/chunks"}, - }, chunk.StoreConfig{}, chunk.SchemaConfig{ - Configs: []chunk.PeriodConfig{ - chunk.PeriodConfig{ - From: chunk.DayTime{Time: start}, - IndexType: "boltdb", - ObjectType: "filesystem", - Schema: "v9", - IndexTables: chunk.PeriodicTableConfig{ - Prefix: "index_", - Period: time.Hour * 168, + store, err := lstore.NewStore( + storage.Config{ + BoltDBConfig: local.BoltDBConfig{Directory: "/tmp/benchmark/index"}, + FSConfig: local.FSConfig{Directory: "/tmp/benchmark/chunks"}, + }, + chunk.StoreConfig{}, + chunk.SchemaConfig{ + Configs: []chunk.PeriodConfig{ + { + From: chunk.DayTime{Time: start}, + IndexType: "boltdb", + ObjectType: "filesystem", + Schema: "v9", + IndexTables: chunk.PeriodicTableConfig{ + Prefix: "index_", + Period: time.Hour * 168, + }, }, }, }, - }, &validation.Overrides{}) + &validation.Overrides{}, + ) if err != nil { return nil, err } @@ -92,7 +99,7 @@ func fillStore() error { Line: randString(250), } if chunkEnc.SpaceFor(entry) { - chunkEnc.Append(entry) + _ = chunkEnc.Append(entry) } else { from, to := chunkEnc.Bounds() c := chunk.NewChunk("fake", fp, metric, chunkenc.NewFacade(chunkEnc), model.TimeFromUnixNano(from.UnixNano()), model.TimeFromUnixNano(to.UnixNano())) diff --git a/pkg/storage/store.go b/pkg/storage/store.go index f89fe7d066e3..dd7286830b80 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -157,6 +157,7 @@ func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][ } iterators = append(iterators, iterator) } + // reduce swap in the heap iterator. if req.Direction == logproto.FORWARD { for i, j := 0, len(iterators)-1; i < j; i, j = i+1, j-1 { iterators[i], iterators[j] = iterators[j], iterators[i] diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index 35dfa52deb5b..7f86051858d8 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -4,7 +4,6 @@ import ( "context" "log" "runtime" - "runtime/debug" "testing" "time" @@ -83,11 +82,11 @@ func Benchmark_store_LazyQueryBackward(b *testing.B) { func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { b.ReportAllocs() - // force to run gc 10x more often - debug.SetGCPercent(10) + // force to run gc 10x more often this can be usefull to detect fast allocation vs leak. + //debug.SetGCPercent(10) stop := make(chan struct{}) go func() { - http.ListenAndServe(":6060", http.DefaultServeMux) + _ = http.ListenAndServe(":6060", http.DefaultServeMux) }() go func() { ticker := time.NewTicker(time.Millisecond) @@ -114,7 +113,7 @@ func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { j++ printHeap(b, false) res = append(res, iter.Entry()) - // todo this should be done in the store. + // limit result by 1000 like the querier would do. if j == 1000 { break } @@ -144,7 +143,7 @@ func getStore() Store { FSConfig: local.FSConfig{Directory: "/tmp/benchmark/chunks"}, }, chunk.StoreConfig{}, chunk.SchemaConfig{ Configs: []chunk.PeriodConfig{ - chunk.PeriodConfig{ + { From: chunk.DayTime{Time: start}, IndexType: "boltdb", ObjectType: "filesystem", diff --git a/pkg/util/conv.go b/pkg/util/conv.go index 084885177541..bc95638cb8ee 100644 --- a/pkg/util/conv.go +++ b/pkg/util/conv.go @@ -1,19 +1,36 @@ package util import ( + "sort" + "strings" + "github.com/cortexproject/cortex/pkg/ingester/client" + "github.com/grafana/loki/pkg/logql" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql" ) +type byLabel []client.LabelAdapter + +func (s byLabel) Len() int { return len(s) } +func (s byLabel) Less(i, j int) bool { return strings.Compare(s[i].Name, s[j].Name) < 0 } +func (s byLabel) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + // ToClientLabels parses the labels and converts them to the Cortex type. func ToClientLabels(labels string) ([]client.LabelAdapter, error) { - ls, err := promql.ParseMetric(labels) + ls, err := logql.ParseExpr(labels) if err != nil { return nil, err } - - return client.FromLabelsToLabelAdapaters(ls), nil + matchers := ls.Matchers() + result := make([]client.LabelAdapter, 0, len(matchers)) + for _, m := range matchers { + result = append(result, client.LabelAdapter{ + Name: m.Name, + Value: m.Value, + }) + } + sort.Sort(byLabel(result)) + return result, nil } // ModelLabelSetToMap convert a model.LabelSet to a map[string]string diff --git a/production/helm/loki/templates/statefulset.yaml b/production/helm/loki/templates/statefulset.yaml index 061f67569e5b..a31af520dbac 100644 --- a/production/helm/loki/templates/statefulset.yaml +++ b/production/helm/loki/templates/statefulset.yaml @@ -75,6 +75,8 @@ spec: - name: JAEGER_AGENT_HOST value: "{{ .Values.tracing.jaegerAgentHost }}" {{- end }} + - name: GOGC + value: "{{ .Values.goGC }}" nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} affinity: diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index c0730abce92f..954e62f41fcc 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -14,6 +14,8 @@ affinity: {} ## StatefulSet annotations annotations: {} +goGC: 100 + # enable tracing for debug, need install jaeger and specify right jaeger_agent_host tracing: jaegerAgentHost: @@ -118,7 +120,7 @@ readinessProbe: port: http-metrics initialDelaySeconds: 90 -replicas: 3 +replicas: 1 resources: {} # limits: From dfb6bcbfa25120affa56af621c1e04315d8e6675 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 3 Jul 2019 12:03:02 -0400 Subject: [PATCH 12/25] fixes typo --- Makefile | 2 +- pkg/storage/store_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index de42793b655a..7f71f379d868 100644 --- a/Makefile +++ b/Makefile @@ -315,4 +315,4 @@ enable-plugin: benchmark-store: go run ./pkg/storage/hack/main.go - go test ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out \ No newline at end of file + go test ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index 7f86051858d8..db4ef0a1656c 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -82,7 +82,7 @@ func Benchmark_store_LazyQueryBackward(b *testing.B) { func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { b.ReportAllocs() - // force to run gc 10x more often this can be usefull to detect fast allocation vs leak. + // force to run gc 10x more often this can be useful to detect fast allocation vs leak. //debug.SetGCPercent(10) stop := make(chan struct{}) go func() { From 536e39d6080a698f2d9ba507ddd7babce92a901e Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 4 Jul 2019 17:06:17 -0400 Subject: [PATCH 13/25] found the memory leak and fixed it. --- pkg/chunkenc/gzip.go | 6 +++++ pkg/chunkenc/lazy_chunk.go | 4 --- pkg/storage/store.go | 53 +++++++++++++++++++++++++------------- pkg/storage/store_test.go | 7 ++--- 4 files changed, 45 insertions(+), 25 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index bf7caa50713b..c485f5c766e2 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -430,6 +430,10 @@ func (hb *headBlock) iterator(mint, maxt int64, filter logql.Filter) iter.EntryI } } + if len(entries) == 0 { + return emptyIterator + } + return &listIterator{ entries: entries, } @@ -553,6 +557,7 @@ func (si *bufferedIterator) Entry() logproto.Entry { } func (si *bufferedIterator) Error() error { return si.err } + func (si *bufferedIterator) Close() error { if !si.closed { si.closed = true @@ -566,4 +571,5 @@ func (si *bufferedIterator) Close() error { } return si.err } + func (si *bufferedIterator) Labels() string { return "" } diff --git a/pkg/chunkenc/lazy_chunk.go b/pkg/chunkenc/lazy_chunk.go index 9135f69fb3e9..cf710bfe5cff 100644 --- a/pkg/chunkenc/lazy_chunk.go +++ b/pkg/chunkenc/lazy_chunk.go @@ -21,8 +21,6 @@ func (c *LazyChunk) getChunk(ctx context.Context) (Chunk, error) { if err != nil { return nil, err } - - c.Chunk = chunks[0] return chunks[0].Data.(*Facade).LokiChunk(), nil } @@ -82,7 +80,6 @@ func (it *lazyIterator) Next() bool { return false } it.EntryIterator, it.err = chk.Iterator(it.from, it.through, it.direction, it.filter) - it.chunk = nil return it.Next() } @@ -102,7 +99,6 @@ func (it *lazyIterator) Error() error { func (it *lazyIterator) Close() error { if it.EntryIterator != nil { - it.chunk = nil it.closed = true err := it.EntryIterator.Close() it.EntryIterator = nil diff --git a/pkg/storage/store.go b/pkg/storage/store.go index dd7286830b80..64f5e6f80792 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -81,13 +81,13 @@ func (s *store) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter // Make sure the initial chunks are loaded. This is not one chunk // per series, but rather a chunk per non-overlapping iterator. - if err := loadFirstChunks(ctx, chksBySeries); err != nil { + if err := loadFirstChunks(ctx, chksBySeries, req); err != nil { return nil, err } // Now that we have the first chunk for each series loaded, // we can proceed to filter the series that don't match. - chksBySeries = filterSeriesByMatchers(chksBySeries, matchers) + chksBySeries = filterSeriesByMatchers(chksBySeries, matchers, req) iters, err := buildIterators(ctx, req, chksBySeries, filter) if err != nil { @@ -111,14 +111,23 @@ func filterChunksByTime(from, through model.Time, chunks []chunk.Chunk) []chunk. return filtered } -func filterSeriesByMatchers(chks map[model.Fingerprint][][]chunkenc.LazyChunk, matchers []*labels.Matcher) map[model.Fingerprint][][]chunkenc.LazyChunk { +func filterSeriesByMatchers(chks map[model.Fingerprint][][]chunkenc.LazyChunk, matchers []*labels.Matcher, req *logproto.QueryRequest) map[model.Fingerprint][][]chunkenc.LazyChunk { outer: for fp, chunks := range chks { for _, matcher := range matchers { - if !matcher.Matches(chunks[0][0].Chunk.Metric.Get(matcher.Name)) { - delete(chks, fp) - continue outer + // checks matchers against the last chunk if we're doing BACKWARD + if req.Direction == logproto.BACKWARD { + if !matcher.Matches(chunks[0][len(chunks[0])-1].Chunk.Metric.Get(matcher.Name)) { + delete(chks, fp) + continue outer + } + } else { + if !matcher.Matches(chunks[0][0].Chunk.Metric.Get(matcher.Name)) { + delete(chks, fp) + continue outer + } } + } } @@ -132,21 +141,24 @@ func buildIterators(ctx context.Context, req *logproto.QueryRequest, chks map[mo if err != nil { return nil, err } - - result = append(result, iterator...) + result = append(result, iterator) } return result, nil } -func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][]chunkenc.LazyChunk, filter logql.Filter) ([]iter.EntryIterator, error) { +func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][]chunkenc.LazyChunk, filter logql.Filter) (iter.EntryIterator, error) { result := make([]iter.EntryIterator, 0, len(chks)) - if chks[0][0].Chunk.Metric.Has("__name__") { - labelsBuilder := labels.NewBuilder(chks[0][0].Chunk.Metric) + var fetchedChunkIndex int + if req.Direction == logproto.BACKWARD { + fetchedChunkIndex = len(chks[0]) - 1 + } + if chks[0][fetchedChunkIndex].Chunk.Metric.Has("__name__") { + labelsBuilder := labels.NewBuilder(chks[0][fetchedChunkIndex].Chunk.Metric) labelsBuilder.Del("__name__") - chks[0][0].Chunk.Metric = labelsBuilder.Labels() + chks[0][fetchedChunkIndex].Chunk.Metric = labelsBuilder.Labels() } - labels := chks[0][0].Chunk.Metric.String() + labels := chks[0][fetchedChunkIndex].Chunk.Metric.String() for i := range chks { iterators := make([]iter.EntryIterator, 0, len(chks[i])) @@ -157,8 +169,8 @@ func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][ } iterators = append(iterators, iterator) } - // reduce swap in the heap iterator. - if req.Direction == logproto.FORWARD { + // reverse chunks to start with the last one. + if req.Direction == logproto.BACKWARD { for i, j := 0, len(iterators)-1; i < j; i, j = i+1, j-1 { iterators[i], iterators[j] = iterators[j], iterators[i] } @@ -166,10 +178,10 @@ func buildHeapIterator(ctx context.Context, req *logproto.QueryRequest, chks [][ result = append(result, iter.NewNonOverlappingIterator(iterators, labels)) } - return result, nil + return iter.NewHeapIterator(result, req.Direction), nil } -func loadFirstChunks(ctx context.Context, chks map[model.Fingerprint][][]chunkenc.LazyChunk) error { +func loadFirstChunks(ctx context.Context, chks map[model.Fingerprint][][]chunkenc.LazyChunk, req *logproto.QueryRequest) error { sp, ctx := opentracing.StartSpanFromContext(ctx, "loadFirstChunks") defer sp.Finish() @@ -180,7 +192,12 @@ func loadFirstChunks(ctx context.Context, chks map[model.Fingerprint][][]chunken if len(lchk) == 0 { continue } - chksByFetcher[lchk[0].Fetcher] = append(chksByFetcher[lchk[0].Fetcher], &lchk[0]) + // load the last chunk if we're doing BACKWARD + if req.Direction == logproto.BACKWARD { + chksByFetcher[lchk[0].Fetcher] = append(chksByFetcher[lchk[0].Fetcher], &lchk[len(lchk)-1]) + } else { + chksByFetcher[lchk[0].Fetcher] = append(chksByFetcher[lchk[0].Fetcher], &lchk[0]) + } } } diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index db4ef0a1656c..28f9c0da7b45 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -108,13 +108,13 @@ func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { } res := []logproto.Entry{} printHeap(b, true) - j := 0 + j := uint32(0) for iter.Next() { j++ printHeap(b, false) res = append(res, iter.Entry()) - // limit result by 1000 like the querier would do. - if j == 1000 { + // limit result like the querier would do. + if j == query.Limit { break } } @@ -134,6 +134,7 @@ func printHeap(b *testing.B, show bool) { } if show { log.Printf("Benchmark %d maxHeapInuse: %d Mbytes\n", b.N, maxHeapInuse/1024/1024) + log.Printf("Benchmark %d currentHeapInuse: %d Mbytes\n", b.N, m.HeapInuse/1024/1024) } } From f123d30e3e8693832cc726774874e11bc671232c Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Fri, 5 Jul 2019 10:39:10 -0400 Subject: [PATCH 14/25] add comment for public interfaces --- pkg/chunkenc/pool.go | 11 ++++++++++- pkg/storage/store.go | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/chunkenc/pool.go b/pkg/chunkenc/pool.go index e1da7ed9645d..fe71b985baf1 100644 --- a/pkg/chunkenc/pool.go +++ b/pkg/chunkenc/pool.go @@ -8,6 +8,8 @@ import ( "sync" ) +// CompressionPool is a pool of CompressionWriter and CompressionReader +// This is used by every chunk to avoid unnecessary allocations. type CompressionPool interface { GetWriter(io.Writer) CompressionWriter PutWriter(CompressionWriter) @@ -16,7 +18,9 @@ type CompressionPool interface { } var ( - Gzip GzipPool + // Gzip is the gun zip compression pool + Gzip GzipPool + // BufReaderPool is bufio.Reader pool BufReaderPool = &BufioReaderPool{ pool: sync.Pool{ New: func() interface{} { return bufio.NewReader(nil) }, @@ -24,11 +28,13 @@ var ( } ) +// GzipPool is a gun zip compression pool type GzipPool struct { readers sync.Pool writers sync.Pool } +// GetReader gets or creates a new CompressionReader and reset it to read from src func (pool *GzipPool) GetReader(src io.Reader) (reader CompressionReader) { if r := pool.readers.Get(); r != nil { reader = r.(CompressionReader) @@ -46,10 +52,12 @@ func (pool *GzipPool) GetReader(src io.Reader) (reader CompressionReader) { return reader } +// PutReader places back in the pool a CompressionReader func (pool *GzipPool) PutReader(reader CompressionReader) { pool.readers.Put(reader) } +// GetWriter gets or creates a new CompressionWriter and reset it to write to dst func (pool *GzipPool) GetWriter(dst io.Writer) (writer CompressionWriter) { if w := pool.writers.Get(); w != nil { writer = w.(CompressionWriter) @@ -60,6 +68,7 @@ func (pool *GzipPool) GetWriter(dst io.Writer) (writer CompressionWriter) { return writer } +// PutWriter places back in the pool a CompressionWriter func (pool *GzipPool) PutWriter(writer CompressionWriter) { pool.writers.Put(writer) } diff --git a/pkg/storage/store.go b/pkg/storage/store.go index 64f5e6f80792..583c63ed28e4 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" ) +// Store is the Loki chunk store to retrieve and save chunks. type Store interface { chunk.Store IsLocal() bool @@ -27,6 +28,7 @@ type store struct { isLocal bool } +// NewStore creates a new Loki Store using configuration supplied. func NewStore(cfg storage.Config, storeCfg chunk.StoreConfig, schemaCfg chunk.SchemaConfig, limits *validation.Overrides) (Store, error) { s, err := storage.NewStore(cfg, storeCfg, schemaCfg, limits) if err != nil { @@ -50,6 +52,8 @@ func (s *store) IsLocal() bool { return s.isLocal } +// LazyQuery returns an iterator that will query the store for more chunks while iterating instead of fetching all chunks upfront +// for that request. func (s *store) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) { expr, err := logql.ParseExpr(req.Query) if err != nil { From ff99471a5467126f8bfdc6be4d06255045716ad9 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Fri, 5 Jul 2019 15:32:43 -0400 Subject: [PATCH 15/25] fixes labels.MatchNotEqual --- pkg/logql/ast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index eaa683459882..83e40a90cf3d 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -89,7 +89,7 @@ func (e *filterExpr) filter() (func([]byte) bool, error) { case labels.MatchNotEqual: f = func(line []byte) bool { - return bytes.Contains(line, []byte(e.match)) + return !bytes.Contains(line, []byte(e.match)) } default: From 059805aaac31c1f52cb2fbc1e0d067ddbffbdd90 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Fri, 5 Jul 2019 23:58:22 -0400 Subject: [PATCH 16/25] add buffer for line in gzip --- pkg/chunkenc/gzip.go | 5 +++-- pkg/chunkenc/pool.go | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index c485f5c766e2..9df1c9b76ca3 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -493,7 +493,7 @@ func newBufferedIterator(pool CompressionPool, b []byte, filter logql.Filter) *b reader: r, pool: pool, filter: filter, - buf: make([]byte, 256), + buf: BytesBufferPool.Get(), decBuf: make([]byte, binary.MaxVarintLen64), } } @@ -533,7 +533,7 @@ func (si *bufferedIterator) moveNext() (int64, []byte, bool) { } for len(si.buf) < int(l) { - si.buf = append(si.buf, make([]byte, 256)...) + si.buf = append(si.buf, make([]byte, 1024)...) } n, err := si.s.Read(si.buf[:l]) @@ -563,6 +563,7 @@ func (si *bufferedIterator) Close() error { si.closed = true si.pool.PutReader(si.reader) BufReaderPool.Put(si.s) + BytesBufferPool.Put(si.buf) si.s = nil si.buf = nil si.decBuf = nil diff --git a/pkg/chunkenc/pool.go b/pkg/chunkenc/pool.go index fe71b985baf1..12995c667ed5 100644 --- a/pkg/chunkenc/pool.go +++ b/pkg/chunkenc/pool.go @@ -26,6 +26,8 @@ var ( New: func() interface{} { return bufio.NewReader(nil) }, }, } + // BytesBufferPool is a bytes buffer used for lines decompressed. + BytesBufferPool = newBufferPoolWithSize(1024) ) // GzipPool is a gun zip compression pool @@ -89,3 +91,24 @@ func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader { func (bufPool *BufioReaderPool) Put(b *bufio.Reader) { bufPool.pool.Put(b) } + + +type bufferPool struct { + pool sync.Pool +} + +func newBufferPoolWithSize(size int) *bufferPool { + return &bufferPool{ + pool: sync.Pool{ + New: func() interface{} { return make([]byte, size) }, + }, + } +} + +func (bp *bufferPool) Get() []byte { + return bp.pool.Get().([]byte) +} + +func (bp *bufferPool) Put(b []byte) { + bp.pool.Put(b) +} From 3272df5cfb2bcc798dfb442b91ce3922f5f99c10 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Sat, 6 Jul 2019 01:09:42 -0400 Subject: [PATCH 17/25] tweak line buffer --- pkg/chunkenc/gzip.go | 14 +++++++------- pkg/chunkenc/pool.go | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 9df1c9b76ca3..e6bca1c97829 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -478,8 +478,8 @@ type bufferedIterator struct { err error - buf []byte // The buffer a single entry. - decBuf []byte // The buffer for decoding the lengths. + buf *bytes.Buffer // The buffer a single entry. + decBuf []byte // The buffer for decoding the lengths. closed bool @@ -532,24 +532,24 @@ func (si *bufferedIterator) moveNext() (int64, []byte, bool) { } } - for len(si.buf) < int(l) { - si.buf = append(si.buf, make([]byte, 1024)...) + for si.buf.Cap() < int(l) { + si.buf.Grow(1024) } - n, err := si.s.Read(si.buf[:l]) + n, err := si.s.Read(si.buf.Bytes()[:l]) if err != nil && err != io.EOF { si.err = err return 0, nil, false } if n < int(l) { - _, err = si.s.Read(si.buf[n:l]) + _, err = si.s.Read(si.buf.Bytes()[n:l]) if err != nil { si.err = err return 0, nil, false } } - return ts, si.buf[:l], true + return ts, si.buf.Bytes()[:l], true } func (si *bufferedIterator) Entry() logproto.Entry { diff --git a/pkg/chunkenc/pool.go b/pkg/chunkenc/pool.go index 12995c667ed5..3471a254a3bb 100644 --- a/pkg/chunkenc/pool.go +++ b/pkg/chunkenc/pool.go @@ -2,6 +2,7 @@ package chunkenc import ( "bufio" + "bytes" "compress/gzip" "io" @@ -27,7 +28,7 @@ var ( }, } // BytesBufferPool is a bytes buffer used for lines decompressed. - BytesBufferPool = newBufferPoolWithSize(1024) + BytesBufferPool = newBufferPoolWithSize(1024) ) // GzipPool is a gun zip compression pool @@ -92,7 +93,6 @@ func (bufPool *BufioReaderPool) Put(b *bufio.Reader) { bufPool.pool.Put(b) } - type bufferPool struct { pool sync.Pool } @@ -100,15 +100,15 @@ type bufferPool struct { func newBufferPoolWithSize(size int) *bufferPool { return &bufferPool{ pool: sync.Pool{ - New: func() interface{} { return make([]byte, size) }, + New: func() interface{} { return bytes.NewBuffer(make([]byte, size)) }, }, } } -func (bp *bufferPool) Get() []byte { - return bp.pool.Get().([]byte) +func (bp *bufferPool) Get() *bytes.Buffer { + return bp.pool.Get().(*bytes.Buffer) } -func (bp *bufferPool) Put(b []byte) { +func (bp *bufferPool) Put(b *bytes.Buffer) { bp.pool.Put(b) } From 940d1667dd80386223633335e992f1290375f9e2 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Sat, 6 Jul 2019 21:47:18 -0400 Subject: [PATCH 18/25] tweak line buffer pool --- pkg/chunkenc/gzip.go | 12 ++++++------ pkg/chunkenc/pool.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index e6bca1c97829..910936e400d5 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -9,11 +9,9 @@ import ( "io" "time" + "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" - - "github.com/grafana/loki/pkg/iter" - "github.com/pkg/errors" ) @@ -493,7 +491,6 @@ func newBufferedIterator(pool CompressionPool, b []byte, filter logql.Filter) *b reader: r, pool: pool, filter: filter, - buf: BytesBufferPool.Get(), decBuf: make([]byte, binary.MaxVarintLen64), } } @@ -532,8 +529,12 @@ func (si *bufferedIterator) moveNext() (int64, []byte, bool) { } } + if si.buf == nil { + si.buf = BytesBufferPool.Get() + } + for si.buf.Cap() < int(l) { - si.buf.Grow(1024) + si.buf.Grow(int(l) - si.buf.Cap()) } n, err := si.s.Read(si.buf.Bytes()[:l]) @@ -548,7 +549,6 @@ func (si *bufferedIterator) moveNext() (int64, []byte, bool) { return 0, nil, false } } - return ts, si.buf.Bytes()[:l], true } diff --git a/pkg/chunkenc/pool.go b/pkg/chunkenc/pool.go index 3471a254a3bb..a843159de25f 100644 --- a/pkg/chunkenc/pool.go +++ b/pkg/chunkenc/pool.go @@ -28,7 +28,7 @@ var ( }, } // BytesBufferPool is a bytes buffer used for lines decompressed. - BytesBufferPool = newBufferPoolWithSize(1024) + BytesBufferPool = newBufferPoolWithSize(4096) ) // GzipPool is a gun zip compression pool From 5146aef3ebc9fd5f780d05a3057e4d79b4a7b2c2 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Sun, 7 Jul 2019 22:20:03 -0400 Subject: [PATCH 19/25] iterator improvement --- pkg/iter/iterator.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pkg/iter/iterator.go b/pkg/iter/iterator.go index 4c2f33a0c9b8..07afa6ef193c 100644 --- a/pkg/iter/iterator.go +++ b/pkg/iter/iterator.go @@ -348,13 +348,11 @@ func (i *nonOverlappingIterator) Next() bool { if len(i.iterators) == 0 { if i.curr != nil { i.curr.Close() - i.curr = nil } return false } if i.curr != nil { i.curr.Close() - i.curr = nil } i.i++ i.curr, i.iterators = i.iterators[0], i.iterators[1:] @@ -364,12 +362,7 @@ func (i *nonOverlappingIterator) Next() bool { } func (i *nonOverlappingIterator) Entry() logproto.Entry { - if i.curr == nil { - return *i.lastEntry - } - entry := i.curr.Entry() - i.lastEntry = &entry - return *i.lastEntry + return i.curr.Entry() } func (i *nonOverlappingIterator) Labels() string { @@ -392,7 +385,6 @@ func (i *nonOverlappingIterator) Close() error { iter.Close() } i.iterators = nil - i.curr = nil return nil } @@ -441,14 +433,15 @@ type entryIteratorBackward struct { // NewEntryIteratorBackward returns an iterator which loads all the entries // of an existing iterator, and then iterates over them backward. func NewEntryIteratorBackward(it EntryIterator) (EntryIterator, error) { - return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 128), forwardIter: it}, it.Error() + return &entryIteratorBackward{entries: make([]logproto.Entry, 0, 1024), forwardIter: it}, it.Error() } func (i *entryIteratorBackward) load() { if !i.loaded { i.loaded = true for i.forwardIter.Next() { - i.entries = append(i.entries, i.forwardIter.Entry()) + entry := i.forwardIter.Entry() + i.entries = append(i.entries, entry) } i.forwardIter.Close() } From a51b1d4b12ab7043a4b9220f08fe7e1058006886 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 8 Jul 2019 08:29:49 -0400 Subject: [PATCH 20/25] fix linter --- pkg/iter/iterator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/iter/iterator.go b/pkg/iter/iterator.go index 07afa6ef193c..dfe3287aa16c 100644 --- a/pkg/iter/iterator.go +++ b/pkg/iter/iterator.go @@ -331,8 +331,6 @@ type nonOverlappingIterator struct { i int iterators []EntryIterator curr EntryIterator - - lastEntry *logproto.Entry } // NewNonOverlappingIterator gives a chained iterator over a list of iterators. From a636b3065cd3c8e91fd53a637c624aec18799e27 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 8 Jul 2019 09:15:37 -0400 Subject: [PATCH 21/25] fix block append --- pkg/chunkenc/gzip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 910936e400d5..237904cd5a51 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -184,7 +184,7 @@ func NewByteChunk(b []byte) (*MemChunk, error) { // Read the number of blocks. num := db.uvarint() - bc.blocks = make([]block, num) + bc.blocks = make([]block, 0, num) for i := 0; i < num; i++ { blk := block{} From f0eea652c800721db2c68430658d8e75498cd3c7 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 15 Jul 2019 09:57:38 -0400 Subject: [PATCH 22/25] remove helm changes and fix some PR feedback --- pkg/chunkenc/dumb_chunk.go | 2 +- pkg/chunkenc/gzip.go | 9 +- pkg/chunkenc/gzip_test.go | 1 + production/helm/loki/Chart.yaml | 2 +- production/helm/loki/templates/_helpers.tpl | 20 -- .../helm/loki/templates/consul/configmap.yaml | 219 ------------------ .../loki/templates/consul/deployment.yaml | 129 ----------- .../helm/loki/templates/consul/service.yaml | 43 ---- .../helm/loki/templates/networkpolicy.yaml | 2 - production/helm/loki/templates/pdb.yaml | 1 - production/helm/loki/templates/secret.yaml | 2 +- .../helm/loki/templates/service-headless.yaml | 1 - production/helm/loki/templates/service.yaml | 1 - .../helm/loki/templates/statefulset.yaml | 5 - production/helm/loki/values.yaml | 58 ++--- 15 files changed, 21 insertions(+), 474 deletions(-) delete mode 100644 production/helm/loki/templates/consul/configmap.yaml delete mode 100644 production/helm/loki/templates/consul/deployment.yaml delete mode 100644 production/helm/loki/templates/consul/service.yaml diff --git a/pkg/chunkenc/dumb_chunk.go b/pkg/chunkenc/dumb_chunk.go index 19446634f7cf..89a3b4128fee 100644 --- a/pkg/chunkenc/dumb_chunk.go +++ b/pkg/chunkenc/dumb_chunk.go @@ -52,7 +52,7 @@ func (c *dumbChunk) Size() int { // Returns an iterator that goes from _most_ recent to _least_ recent (ie, // backwards). -func (c *dumbChunk) Iterator(from, through time.Time, direction logproto.Direction, filter logql.Filter) (iter.EntryIterator, error) { +func (c *dumbChunk) Iterator(from, through time.Time, direction logproto.Direction, _ logql.Filter) (iter.EntryIterator, error) { i := sort.Search(len(c.entries), func(i int) bool { return !from.After(c.entries[i].Timestamp) }) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 237904cd5a51..eb2e439675d3 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -476,7 +476,7 @@ type bufferedIterator struct { err error - buf *bytes.Buffer // The buffer a single entry. + buf *bytes.Buffer // The buffer for a single entry. decBuf []byte // The buffer for decoding the lengths. closed bool @@ -491,6 +491,7 @@ func newBufferedIterator(pool CompressionPool, b []byte, filter logql.Filter) *b reader: r, pool: pool, filter: filter, + buf: BytesBufferPool.Get(), decBuf: make([]byte, binary.MaxVarintLen64), } } @@ -529,11 +530,7 @@ func (si *bufferedIterator) moveNext() (int64, []byte, bool) { } } - if si.buf == nil { - si.buf = BytesBufferPool.Get() - } - - for si.buf.Cap() < int(l) { + if si.buf.Cap() < int(l) { si.buf.Grow(int(l) - si.buf.Cap()) } diff --git a/pkg/chunkenc/gzip_test.go b/pkg/chunkenc/gzip_test.go index bb52477c5e83..89e701c33d3d 100644 --- a/pkg/chunkenc/gzip_test.go +++ b/pkg/chunkenc/gzip_test.go @@ -260,6 +260,7 @@ func BenchmarkReadGZIP(b *testing.B) { chunks = append(chunks, c) } entries := []logproto.Entry{} + b.ResetTimer() for n := 0; n < b.N; n++ { var wg sync.WaitGroup for _, c := range chunks { diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 4be7c928c860..8703a5fff724 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.11.0 +version: 0.10.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl index 02b5713715da..2e333aae60f8 100644 --- a/production/helm/loki/templates/_helpers.tpl +++ b/production/helm/loki/templates/_helpers.tpl @@ -41,23 +41,3 @@ Create the name of the service account {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} - -{{/* -Create a config to merge with the current loki config to overrides values -*/}} -{{- define "loki.config.overrides" -}} -ingester: - lifecycler: - ring: - kvstore: - {{ if (include "loki.deployConsul" .)}} - consul: - host: {{ printf "%s-consul-svc.%s.svc.cluster.local:8500" (include "loki.fullname" .) .Release.Namespace}} - {{end}} -{{- end -}} - -{{- define "loki.deployConsul" -}} - {{- if and (eq .Values.config.ingester.lifecycler.ring.kvstore.store "consul") .Values.consul.enabled -}} - {{true}} - {{- end -}} -{{- end -}} diff --git a/production/helm/loki/templates/consul/configmap.yaml b/production/helm/loki/templates/consul/configmap.yaml deleted file mode 100644 index 70a4d6935d24..000000000000 --- a/production/helm/loki/templates/consul/configmap.yaml +++ /dev/null @@ -1,219 +0,0 @@ - {{- if (include "loki.deployConsul" .) }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "loki.fullname" . }}-consul-config - labels: - name: consul-config - app: {{ template "loki.name" . }} - chart: {{ template "loki.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - consul-config.json: "{\"leave_on_terminate\": true, \"telemetry\": {\"dogstatsd_addr\": \"127.0.0.1:9125\"}}\n" - consul.config: "{\"leave_on_terminate\": true, \"telemetry\": {\"dogstatsd_addr\": \"127.0.0.1:9125\"}}" - mapping: | - mappings: - - match: consul.*.runtime.* - name: consul_runtime - labels: - type: $2 - - match: consul.runtime.total_gc_pause_ns - name: consul_runtime_total_gc_pause_ns - labels: - type: $2 - - match: consul.consul.health.service.query-tag.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3 - - match: consul.consul.health.service.query-tag.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4 - - match: consul.consul.health.service.query-tag.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5 - - match: consul.consul.health.service.query-tag.*.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5.$6 - - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5.$6.$7 - - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5.$6.$7.$8 - - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5.$6.$7.$8.$9 - - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10 - - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10.$11 - - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.*.*.* - name: consul_health_service_query_tag - labels: - query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10.$11.$12 - - match: consul.consul.catalog.deregister - name: consul_catalog_deregister - labels: {} - - match: consul.consul.dns.domain_query.*.*.*.*.* - name: consul_dns_domain_query - labels: - query: $1.$2.$3.$4.$5 - - match: consul.consul.health.service.not-found.* - name: consul_health_service_not_found - labels: - query: $1 - - match: consul.consul.health.service.query.* - name: consul_health_service_query - labels: - query: $1 - - match: consul.*.memberlist.health.score - name: consul_memberlist_health_score - labels: {} - - match: consul.serf.queue.* - name: consul_serf_events - labels: - type: $1 - - match: consul.serf.snapshot.appendLine - name: consul_serf_snapshot_appendLine - labels: - type: $1 - - match: consul.serf.coordinate.adjustment-ms - name: consul_serf_coordinate_adjustment_ms - labels: {} - - match: consul.consul.rpc.query - name: consul_rpc_query - labels: {} - - match: consul.*.consul.session_ttl.active - name: consul_session_ttl_active - labels: {} - - match: consul.raft.rpc.* - name: consul_raft_rpc - labels: - type: $1 - - match: consul.raft.rpc.appendEntries.storeLogs - name: consul_raft_rpc_appendEntries_storeLogs - labels: - type: $1 - - match: consul.consul.fsm.persist - name: consul_fsm_persist - labels: {} - - match: consul.raft.fsm.apply - name: consul_raft_fsm_apply - labels: {} - - match: consul.raft.leader.lastContact - name: consul_raft_leader_lastcontact - labels: {} - - match: consul.raft.leader.dispatchLog - name: consul_raft_leader_dispatchLog - labels: {} - - match: consul.raft.commitTime - name: consul_raft_commitTime - labels: {} - - match: consul.raft.replication.appendEntries.logs.*.*.*.* - name: consul_raft_replication_appendEntries_logs - labels: - query: ${1}.${2}.${3}.${4} - - match: consul.raft.replication.appendEntries.rpc.*.*.*.* - name: consul_raft_replication_appendEntries_rpc - labels: - query: ${1}.${2}.${3}.${4} - - match: consul.raft.replication.heartbeat.*.*.*.* - name: consul_raft_replication_heartbeat - labels: - query: ${1}.${2}.${3}.${4} - - match: consul.consul.rpc.request - name: consul_rpc_requests - labels: {} - - match: consul.consul.rpc.accept_conn - name: consul_rpc_accept_conn - labels: {} - - match: consul.memberlist.udp.* - name: consul_memberlist_udp - labels: - type: $1 - - match: consul.memberlist.tcp.* - name: consul_memberlist_tcp - labels: - type: $1 - - match: consul.memberlist.gossip - name: consul_memberlist_gossip - labels: {} - - match: consul.memberlist.probeNode - name: consul_memberlist_probenode - labels: {} - - match: consul.memberlist.pushPullNode - name: consul_memberlist_pushpullnode - labels: {} - - match: consul.http.* - name: consul_http_request - labels: - method: $1 - path: / - - match: consul.http.*.* - name: consul_http_request - labels: - method: $1 - path: /$2 - - match: consul.http.*.*.* - name: consul_http_request - labels: - method: $1 - path: /$2/$3 - - match: consul.http.*.*.*.* - name: consul_http_request - labels: - method: $1 - path: /$2/$3/$4 - - match: consul.http.*.*.*.*.* - name: consul_http_request - labels: - method: $1 - path: /$2/$3/$4/$5 - - match: consul.consul.leader.barrier - name: consul_leader_barrier - labels: {} - - match: consul.consul.leader.reconcileMember - name: consul_leader_reconcileMember - labels: {} - - match: consul.consul.leader.reconcile - name: consul_leader_reconcile - labels: {} - - match: consul.consul.fsm.coordinate.batch-update - name: consul_fsm_coordinate_batch_update - labels: {} - - match: consul.consul.fsm.autopilot - name: consul_fsm_autopilot - labels: {} - - match: consul.consul.fsm.kvs.cas - name: consul_fsm_kvs_cas - labels: {} - - match: consul.consul.fsm.register - name: consul_fsm_register - labels: {} - - match: consul.consul.fsm.deregister - name: consul_fsm_deregister - labels: {} - - match: consul.consul.fsm.tombstone.reap - name: consul_fsm_tombstone_reap - labels: {} - - match: consul.consul.catalog.register - name: consul_catalog_register - labels: {} - - match: consul.consul.catalog.deregister - name: consul_catalog_deregister - labels: {} - - match: consul.consul.leader.reapTombstones - name: consul_leader_reapTombstones - labels: {} -{{- end }} \ No newline at end of file diff --git a/production/helm/loki/templates/consul/deployment.yaml b/production/helm/loki/templates/consul/deployment.yaml deleted file mode 100644 index 799022db5ae1..000000000000 --- a/production/helm/loki/templates/consul/deployment.yaml +++ /dev/null @@ -1,129 +0,0 @@ - {{- if (include "loki.deployConsul" .) }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "loki.fullname" . }}-consul - labels: - name: consul - app: {{ template "loki.name" . }} - chart: {{ template "loki.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - annotations: - {{- toYaml .Values.consul.annotations | nindent 4 }} -spec: - minReadySeconds: 10 - replicas: {{ .Values.consul.replicas }} - selector: - matchLabels: - name: consul - app: {{ template "loki.name" . }} - release: {{ .Release.Name }} - strategy: - {{- toYaml .Values.consul.updateStrategy | nindent 4 }} - template: - metadata: - labels: - name: consul - app: {{ template "loki.name" . }} - release: {{ .Release.Name }} - annotations: - checksum/config: {{ include (print $.Template.BasePath "/consul/configmap.yaml") . | sha256sum }} - {{- with .Values.consul.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - affinity: - {{- toYaml .Values.consul.affinity | nindent 8 }} - nodeSelector: - {{- toYaml .Values.consul.nodeSelector | nindent 8 }} - tolerations: - {{- toYaml .Values.consul.tolerations | nindent 8 }} - containers: - - name: consul - args: - - agent - - -ui - - -server - - -client=0.0.0.0 - - -config-file=/etc/config/consul-config.json - - -bootstrap-expect=1 - env: - - name: CHECKPOINT_DISABLE - value: "1" - image: "{{ .Values.consul.image.repository }}:{{ .Values.consul.image.tag }}" - imagePullPolicy: {{ .Values.consul.image.pullPolicy }} - ports: - - containerPort: 8300 - name: server - protocol: TCP - - containerPort: 8301 - name: serf - protocol: TCP - - containerPort: 8400 - name: client - protocol: TCP - - containerPort: 8500 - name: api - protocol: TCP - resources: - {{- toYaml .Values.consul.resources | nindent 10 }} - volumeMounts: - - mountPath: /etc/config - name: consul - - args: - - --namespace=$(POD_NAMESPACE) - - --pod-name=$(POD_NAME) - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - image: weaveworks/consul-sidekick:master-f18ad13 - imagePullPolicy: IfNotPresent - name: sidekick - resources: {} - volumeMounts: - - mountPath: /etc/config - name: consul - - args: - - --web.listen-address=:8000 - - --statsd.mapping-config=/etc/config/mapping - image: prom/statsd-exporter:v0.8.1 - imagePullPolicy: IfNotPresent - name: statsd-exporter - ports: - - containerPort: 8000 - name: http-metrics - protocol: TCP - resources: {} - volumeMounts: - - mountPath: /etc/config - name: consul - - args: - - --consul.server=localhost:8500 - - --web.listen-address=:9107 - - --consul.timeout=1s - image: prom/consul-exporter:v0.4.0 - imagePullPolicy: IfNotPresent - name: consul-exporter - ports: - - containerPort: 9107 - name: http-metrics - protocol: TCP - resources: {} - volumeMounts: - - mountPath: /etc/config - name: consul - volumes: - - configMap: - defaultMode: 420 - name: {{ template "loki.fullname" . }}-consul-config - name: consul -{{- end }} \ No newline at end of file diff --git a/production/helm/loki/templates/consul/service.yaml b/production/helm/loki/templates/consul/service.yaml deleted file mode 100644 index bd175edba39f..000000000000 --- a/production/helm/loki/templates/consul/service.yaml +++ /dev/null @@ -1,43 +0,0 @@ - {{- if (include "loki.deployConsul" .) }} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "loki.fullname" . }}-consul-svc - labels: - name: consul-svc - app: {{ template "loki.name" . }} - chart: {{ template "loki.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - type: ClusterIP - ports: - - name: consul-server - port: 8300 - protocol: TCP - targetPort: 8300 - - name: consul-serf - port: 8301 - protocol: TCP - targetPort: 8301 - - name: consul-client - port: 8400 - protocol: TCP - targetPort: 8400 - - name: consul-api - port: 8500 - protocol: TCP - targetPort: 8500 - - name: statsd-exporter-http-metrics - port: 8000 - protocol: TCP - targetPort: 8000 - - name: consul-exporter-http-metrics - port: 9107 - protocol: TCP - targetPort: 9107 - selector: - name: consul - app: {{ template "loki.name" . }} - release: {{ .Release.Name }} -{{- end }} \ No newline at end of file diff --git a/production/helm/loki/templates/networkpolicy.yaml b/production/helm/loki/templates/networkpolicy.yaml index 0e71a9bf09aa..5becbcdb5915 100644 --- a/production/helm/loki/templates/networkpolicy.yaml +++ b/production/helm/loki/templates/networkpolicy.yaml @@ -11,7 +11,6 @@ metadata: spec: podSelector: matchLabels: - name: loki name: {{ template "loki.fullname" . }} app: {{ template "loki.name" . }} release: {{ .Release.Name }} @@ -19,7 +18,6 @@ spec: - from: - podSelector: matchLabels: - name: loki app: {{ template "promtail.name" . }} release: {{ .Release.Name }} - ports: diff --git a/production/helm/loki/templates/pdb.yaml b/production/helm/loki/templates/pdb.yaml index 6c3bb54c8c87..27093f5d8ed6 100644 --- a/production/helm/loki/templates/pdb.yaml +++ b/production/helm/loki/templates/pdb.yaml @@ -11,7 +11,6 @@ metadata: spec: selector: matchLabels: - name: loki app: {{ template "loki.name" . }} {{ toYaml .Values.podDisruptionBudget | indent 2 }} {{- end -}} diff --git a/production/helm/loki/templates/secret.yaml b/production/helm/loki/templates/secret.yaml index f52f7a3c6376..95b16fad4be9 100644 --- a/production/helm/loki/templates/secret.yaml +++ b/production/helm/loki/templates/secret.yaml @@ -8,4 +8,4 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - loki.yaml: {{ tpl (toYaml (mergeOverwrite .Values.config (fromYaml (include "loki.config.overrides" . )))) . | b64enc}} + loki.yaml: {{ tpl (toYaml .Values.config) . | b64enc}} diff --git a/production/helm/loki/templates/service-headless.yaml b/production/helm/loki/templates/service-headless.yaml index 3167d861b6a6..dbc127b3bea3 100644 --- a/production/helm/loki/templates/service-headless.yaml +++ b/production/helm/loki/templates/service-headless.yaml @@ -15,6 +15,5 @@ spec: name: http-metrics targetPort: http-metrics selector: - name: loki app: {{ template "loki.name" . }} release: {{ .Release.Name }} diff --git a/production/helm/loki/templates/service.yaml b/production/helm/loki/templates/service.yaml index ebbb11adb195..9a8c1da58592 100644 --- a/production/helm/loki/templates/service.yaml +++ b/production/helm/loki/templates/service.yaml @@ -26,6 +26,5 @@ spec: nodePort: {{ .Values.service.nodePort }} {{- end }} selector: - name: loki app: {{ template "loki.name" . }} release: {{ .Release.Name }} diff --git a/production/helm/loki/templates/statefulset.yaml b/production/helm/loki/templates/statefulset.yaml index a31af520dbac..a1813b130aa2 100644 --- a/production/helm/loki/templates/statefulset.yaml +++ b/production/helm/loki/templates/statefulset.yaml @@ -3,7 +3,6 @@ kind: StatefulSet metadata: name: {{ template "loki.fullname" . }} labels: - name: loki app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} release: {{ .Release.Name }} @@ -15,7 +14,6 @@ spec: replicas: {{ .Values.replicas }} selector: matchLabels: - name: loki app: {{ template "loki.name" . }} release: {{ .Release.Name }} serviceName: {{ template "loki.fullname" . }}-headless @@ -24,7 +22,6 @@ spec: template: metadata: labels: - name: loki app: {{ template "loki.name" . }} name: {{ template "loki.name" . }} release: {{ .Release.Name }} @@ -75,8 +72,6 @@ spec: - name: JAEGER_AGENT_HOST value: "{{ .Values.tracing.jaegerAgentHost }}" {{- end }} - - name: GOGC - value: "{{ .Values.goGC }}" nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} affinity: diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 954e62f41fcc..c6725a5f8b9b 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -14,8 +14,6 @@ affinity: {} ## StatefulSet annotations annotations: {} -goGC: 100 - # enable tracing for debug, need install jaeger and specify right jaeger_agent_host tracing: jaegerAgentHost: @@ -23,18 +21,23 @@ tracing: config: auth_enabled: false ingester: - chunk_idle_period: 5m - chunk_retain_period: 30s + chunk_idle_period: 15m chunk_block_size: 262144 lifecycler: ring: kvstore: - store: consul - consul: - consistentreads: true - httpclienttimeout: 20s - prefix: "" + store: inmemory replication_factor: 1 + + ## Different ring configs can be used. E.g. Consul + # ring: + # store: consul + # replication_factor: 1 + # consul: + # host: "consul:8500" + # prefix: "" + # httpclienttimeout: "20s" + # consistentreads: true limits_config: enforce_metric_name: false reject_old_samples: true @@ -74,7 +77,7 @@ livenessProbe: httpGet: path: /ready port: http-metrics - initialDelaySeconds: 90 + initialDelaySeconds: 45 ## Enable persistence using Persistent Volume Claims networkPolicy: @@ -118,7 +121,7 @@ readinessProbe: httpGet: path: /ready port: http-metrics - initialDelaySeconds: 90 + initialDelaySeconds: 45 replicas: 1 @@ -161,36 +164,3 @@ podDisruptionBudget: {} updateStrategy: type: RollingUpdate - -# consul configuration used by the ring if consul is selected -consul: - # deploy consul if the kvstore is consul, set to false to use your own consul instance. - # The host can be set by using config.ingester.lifecycler.ring.kvstore.consul.host - enabled: true - annotations: {} - replicas: 1 - resources: - requests: - cpu: 100m - memory: 500Mi - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - nodeSelector: {} - tolerations: [] - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - name: consul - topologyKey: kubernetes.io/hostname - podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "http-metrics" - image: - repository: consul - tag: 1.4.0 - pullPolicy: IfNotPresent From 7284917b3be553e6cb3a7295108e2dc312d41b60 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 15 Jul 2019 10:25:10 -0400 Subject: [PATCH 23/25] revert scaling on disk --- pkg/ingester/ingester.go | 38 +---------------------------------- pkg/ingester/ingester_test.go | 8 -------- pkg/ingester/instance.go | 13 ++++++------ pkg/querier/querier.go | 29 +++++++++----------------- pkg/storage/store.go | 17 +--------------- pkg/storage/store_test.go | 2 +- 6 files changed, 18 insertions(+), 89 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 9436b5afb80e..05638f37174d 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -17,8 +17,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/util" - "github.com/grafana/loki/pkg/helpers" - "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" ) @@ -76,8 +74,6 @@ type Ingester struct { // ChunkStore is the interface we need to store chunks. type ChunkStore interface { Put(ctx context.Context, chunks []chunk.Chunk) error - IsLocal() bool - LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) } // New makes a new Ingester. @@ -187,40 +183,8 @@ func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querie return err } - iters := []iter.EntryIterator{} instance := i.getOrCreateInstance(instanceID) - instanceIter, err := instance.Query(req) - if err != nil { - return err - } - iters = append(iters, instanceIter) - - // last sample in memory is either now minus retain period - // or since the instance was created. - lastInmemory := time.Now().Add(-i.cfg.RetainPeriod) - if instance.createdAt.After(lastInmemory) { - lastInmemory = instance.createdAt - } - - // we should also query the store if: - // - it is local to the ingester - // - the request is beyond what the ingester has in memory - if i.store.IsLocal() && req.Start.Before(lastInmemory) { - // we query the remaining data from the store. - if req.End.After(lastInmemory) { - req.End = lastInmemory - } - lazyIter, err := i.store.LazyQuery(queryServer.Context(), req) - if err != nil { - return err - } - iters = append(iters, lazyIter) - } - - iter := iter.NewHeapIterator(iters, req.Direction) - defer helpers.LogError("closing iterator", iter.Close) - - return sendBatches(iter, queryServer, req.Limit) + return instance.Query(req, queryServer) } // Label returns the set of labels for the stream this ingester knows about. diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 5a488d6455f4..16f395b5becb 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" "github.com/stretchr/testify/require" "github.com/weaveworks/common/user" @@ -111,13 +110,6 @@ func (s *mockStore) Put(ctx context.Context, chunks []chunk.Chunk) error { return nil } -func (s *mockStore) IsLocal() bool { - return false -} -func (s *mockStore) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) { - return nil, nil -} - type mockQuerierServer struct { ctx context.Context resps []*logproto.QueryResponse diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index d2e66fdd01a6..be38c6400aaf 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -3,7 +3,6 @@ package ingester import ( "context" "sync" - "time" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -56,7 +55,6 @@ type instance struct { blockSize int tailers map[uint32]*tailer tailerMtx sync.RWMutex - createdAt time.Time } func newInstance(instanceID string, blockSize int) *instance { @@ -70,7 +68,6 @@ func newInstance(instanceID string, blockSize int) *instance { blockSize: blockSize, tailers: map[uint32]*tailer{}, - createdAt: time.Now(), } } @@ -105,10 +102,10 @@ func (i *instance) Push(ctx context.Context, req *logproto.PushRequest) error { return appendErr } -func (i *instance) Query(req *logproto.QueryRequest) (iter.EntryIterator, error) { +func (i *instance) Query(req *logproto.QueryRequest, queryServer logproto.Querier_QueryServer) error { expr, err := logql.ParseExpr(req.Query) if err != nil { - return nil, err + return err } if req.Regex != "" { @@ -126,9 +123,11 @@ func (i *instance) Query(req *logproto.QueryRequest) (iter.EntryIterator, error) iter, err := expr.Eval(querier) if err != nil { - return nil, err + return err } - return iter, nil + defer helpers.LogError("closing iterator", iter.Close) + + return sendBatches(iter, queryServer, req.Limit) } func (i *instance) Label(_ context.Context, req *logproto.LabelRequest) (*logproto.LabelResponse, error) { diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index df3c0a4e01c7..d49c28dea5c1 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -106,24 +106,18 @@ func (q *Querier) forGivenIngesters(replicationSet ring.ReplicationSet, f func(l // Query does the heavy lifting for an actual query. func (q *Querier) Query(ctx context.Context, req *logproto.QueryRequest) (*logproto.QueryResponse, error) { - var iterators []iter.EntryIterator ingesterIterators, err := q.queryIngesters(ctx, req) if err != nil { return nil, err } - iterators = append(iterators, ingesterIterators...) - - // if the store is local to ingester, we want to query it from the ingester only - // the querier doesn't have access. - if !q.store.IsLocal() { - chunkStoreIterators, err := q.store.LazyQuery(ctx, req) - if err != nil { - return nil, err - } - iterators = append(iterators, chunkStoreIterators) + + chunkStoreIterators, err := q.store.LazyQuery(ctx, req) + if err != nil { + return nil, err } + iterators := append(ingesterIterators, chunkStoreIterators) iterator := iter.NewHeapIterator(iterators, req.Direction) defer helpers.LogError("closing iterator", iterator.Close) @@ -284,22 +278,17 @@ func (q *Querier) queryDroppedStreams(ctx context.Context, req *logproto.TailReq return nil, err } - var iterators []iter.EntryIterator ingesterIterators := make([]iter.EntryIterator, len(clients)) for i := range clients { ingesterIterators[i] = iter.NewQueryClientIterator(clients[i].response.(logproto.Querier_QueryClient), query.Direction) } - iterators = append(iterators, ingesterIterators...) - // if the store is local to ingester, we want to query it from the ingester not from the querier - if !q.store.IsLocal() { - chunkStoreIterators, err := q.store.LazyQuery(ctx, &query) - if err != nil { - return nil, err - } - iterators = append(iterators, chunkStoreIterators) + chunkStoreIterators, err := q.store.LazyQuery(ctx, &query) + if err != nil { + return nil, err } + iterators := append(ingesterIterators, chunkStoreIterators) return iter.NewHeapIterator(iterators, query.Direction), nil } diff --git a/pkg/storage/store.go b/pkg/storage/store.go index 583c63ed28e4..0ff8fccae615 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -19,13 +19,11 @@ import ( // Store is the Loki chunk store to retrieve and save chunks. type Store interface { chunk.Store - IsLocal() bool LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) } type store struct { chunk.Store - isLocal bool } // NewStore creates a new Loki Store using configuration supplied. @@ -34,24 +32,11 @@ func NewStore(cfg storage.Config, storeCfg chunk.StoreConfig, schemaCfg chunk.Sc if err != nil { return nil, err } - var isLocal bool - for _, cfg := range schemaCfg.Configs { - if cfg.ObjectType == "filesystem" || cfg.IndexType == "boltdb" { - isLocal = true - break - } - } return &store{ - Store: s, - isLocal: isLocal, + Store: s, }, nil } -// IsLocal tells if the storage for chunks and indexes is local. -func (s *store) IsLocal() bool { - return s.isLocal -} - // LazyQuery returns an iterator that will query the store for more chunks while iterating instead of fetching all chunks upfront // for that request. func (s *store) LazyQuery(ctx context.Context, req *logproto.QueryRequest) (iter.EntryIterator, error) { diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index 28f9c0da7b45..899fa12b74d1 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -107,7 +107,7 @@ func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { b.Fatal(err) } res := []logproto.Entry{} - printHeap(b, true) + printHeap(b, false) j := uint32(0) for iter.Next() { j++ From ab7c337413077b38e520017d60b57d90ac1707d7 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 15 Jul 2019 16:18:03 -0400 Subject: [PATCH 24/25] improve test to use random line length --- pkg/chunkenc/gzip_test.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/pkg/chunkenc/gzip_test.go b/pkg/chunkenc/gzip_test.go index 89e701c33d3d..ead2a175dd90 100644 --- a/pkg/chunkenc/gzip_test.go +++ b/pkg/chunkenc/gzip_test.go @@ -242,20 +242,15 @@ func BenchmarkWriteGZIP(b *testing.B) { func BenchmarkReadGZIP(b *testing.B) { chunks := []Chunk{} - - entry := &logproto.Entry{ - Timestamp: time.Unix(0, 0), - Line: RandString(512), - } i := int64(0) - for n := 0; n < 50; n++ { + entry := randSizeEntry(0) c := NewMemChunk(EncGZIP) // adds until full so we trigger cut which serialize using gzip for c.SpaceFor(entry) { _ = c.Append(entry) - entry.Timestamp = time.Unix(0, i) i++ + entry = randSizeEntry(i) } chunks = append(chunks, c) } @@ -266,7 +261,7 @@ func BenchmarkReadGZIP(b *testing.B) { for _, c := range chunks { wg.Add(1) go func(c Chunk) { - iterator, err := c.Iterator(time.Unix(0, 0), time.Unix(0, 10), logproto.BACKWARD, nil) + iterator, err := c.Iterator(time.Unix(0, 0), time.Now(), logproto.BACKWARD, nil) if err != nil { panic(err) } @@ -282,6 +277,24 @@ func BenchmarkReadGZIP(b *testing.B) { } +func randSizeEntry(ts int64) *logproto.Entry { + var line string + switch ts % 10 { + case 0: + line = RandString(27000) + case 1: + line = RandString(10000) + case 2, 3, 4, 5: + line = RandString(2048) + default: + line = RandString(4096) + } + return &logproto.Entry{ + Timestamp: time.Unix(0, ts), + Line: line, + } +} + const charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" From 76e03e7936744eadb2445c835651f51ac693d822 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 15 Jul 2019 17:04:26 -0400 Subject: [PATCH 25/25] fix a read bug causing blocks read to be totally off --- pkg/chunkenc/gzip.go | 5 +++-- pkg/chunkenc/gzip_test.go | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index eb2e439675d3..ff5bc5d54ce2 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -539,12 +539,13 @@ func (si *bufferedIterator) moveNext() (int64, []byte, bool) { si.err = err return 0, nil, false } - if n < int(l) { - _, err = si.s.Read(si.buf.Bytes()[n:l]) + for n < int(l) { + r, err := si.s.Read(si.buf.Bytes()[n:l]) if err != nil { si.err = err return 0, nil, false } + n += r } return ts, si.buf.Bytes()[:l], true } diff --git a/pkg/chunkenc/gzip_test.go b/pkg/chunkenc/gzip_test.go index ead2a175dd90..5e1363e08b11 100644 --- a/pkg/chunkenc/gzip_test.go +++ b/pkg/chunkenc/gzip_test.go @@ -252,6 +252,7 @@ func BenchmarkReadGZIP(b *testing.B) { i++ entry = randSizeEntry(i) } + c.Close() chunks = append(chunks, c) } entries := []logproto.Entry{}