Sudden high latency #9128

clcc2019 · 2024-08-29T01:18:53Z

clcc2019
Aug 29, 2024

wangjinxiang0522 · 2024-08-30T02:25:33Z

wangjinxiang0522
Aug 30, 2024

Hi, we've just started using Mimir, and our volume is much smaller than yours, but our latency is higher than what's shown in the chart. Can you share your Mimir configuration? Here's my current configuration:

image:
  repository: reg.sprucetec.com/monitor/grafana/mimir
  tag: r281-93e069f
  pullPolicy: IfNotPresent

global:
  dnsService: "kube-dns"
  # extraEnvFrom:
  #   - secretRef:
  #       name: mimir-bucket-secret
  podAnnotations:
    bucketSecretVersion: "0"

# This turns of the built-in MinIO support
minio:
  enabled: false
nginx:
  enabled: false


gateway:
  enabledNonEnterprise: true
  nodeSelector:
    node-type: monitoring
  replicas: 3
  nginxConfig:
    accessLogEnabled: false
  nginx:
    config:
      enableIPv6: false
    verboseLogging: false
    image:
      registry: reg.sprucetec.com
      repository: monitor/docker.io/nginxinc/nginx-unprivileged
      tag: 1.25-alpine
      pullPolicy: IfNotPresent

  service:
    legacyPorts: null
  resources:
    limits:
      memory: "4Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "2"



ingester:
  replicas: 40
  nodeSelector:
    node-type: monitoring
  zoneAwareReplication:
    enabled: false
  env:
    - name: GOMEMLIMIT
      value: "14745MiB"
    - name: GOMAXPROCS
      value: "4"
  resources:
    limits:
      memory: "16Gi"
      cpu: "4"
    requests:
      memory: "5Gi"
      cpu: "500m"
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"



distributor:
  replicas: 25
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "8Gi"
      cpu: "4"
    requests:
      memory: "256Mi"
      cpu: "200m"
  env:
    - name: GOMEMLIMIT
      value: "7372MiB"
    - name: GOMAXPROCS
      value: "4"



querier:
  replicas: 10
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "8Gi"
      cpu: "4"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "7372MiB"
    - name: GOMAXPROCS
      value: "4"


query_frontend:
  replicas: 4
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "100Mi"
      cpu: "200m"
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "4"



ruler:
  enabled: true
  replicas: 4
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "200Mi"
      cpu: "200m"
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "4"

alertmanager:
  nodeSelector:
    node-type: monitoring
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"

  replicas: 2
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "1Gi"
      cpu: "800m"
  statefulSet:
    enabled: true
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "4"

memcachedExporter:
  enabled: true
  image:
    repository: reg.sprucetec.com/monitor/memcached-exporter
    tag: v0.14.3
    pullPolicy: IfNotPresent

memcached:
  image:
    repository: reg.sprucetec.com/monitor/memcached
    tag: 1.6.25-alpine
    pullPolicy: IfNotPresent

admin-cache:
  enabled: true
  replicas: 1
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "16Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "14745MiB"
    - name: GOMAXPROCS
      value: "2"


chunks-cache:
  enabled: true
  replicas: 2
  allocatedMemory: 16384
  maxItemMemory: 10
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "18Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "16588MiB"
    - name: GOMAXPROCS
      value: "2"


index-cache:
  enabled: true
  replicas: 1
  allocatedMemory: 8192
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "10Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "8892MiB"
    - name: GOMAXPROCS
      value: "2"




metadata-cache:
  enabled: true
  nodeSelector:
    node-type: monitoring
  replicas: 1
  allocatedMemory: 1024
  maxItemMemory: 10
  resources:
    limits:
      memory: "2Gi"
      cpu: "1"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "1800MiB"
    - name: GOMAXPROCS
      value: "1"


results-cache:
  nodeSelector:
    node-type: monitoring
  enabled: true
  replicas: 2
  allocatedMemory: 16384
  maxItemMemory: 10
  resources:
    limits:
      memory: "18Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "16588MiB"
    - name: GOMAXPROCS
      value: "2"


store_gateway:
  replicas: 6
  zoneAwareReplication:
    enabled: false
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "16Gi"
      cpu: "6"
    requests:
      memory: "500Mi"
      cpu: "200m"
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"


compactor:
  replicas: 2
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "1Gi"
      cpu: "100m"
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "4"


overrides_exporter:
  enabled: true
  replicas: 2
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "1"
    requests:
      cpu: 50m
      memory: 64Mi
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "1"



query_scheduler:
  enabled: true
  replicas: 2
  nodeSelector:
    node-type: monitoring
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      memory: "4Gi"
      cpu: "2"
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "2"



rollout_operator:
  enabled: true
  image:
    repository: reg.sprucetec.com/monitor/grafana/rollout-operator
    tag: v0.13.0
    pullPolicy: IfNotPresent

  nodeSelector:
    node-type: monitoring
  resources:
    requests:
      cpu: 100m
      memory: 64Mi
    limits:
      memory: "4Gi"
      cpu: "2"
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "2"


mimir:
  structuredConfig:
    multitenancy_enabled: false
    server:
      grpc_server_max_recv_msg_size: 104857600
      grpc_server_max_send_msg_size: 104857600
      grpc_server_max_concurrent_streams: 1500
      log_level: warn
    common:
      storage:
        backend: s3
        s3:
          endpoint: 'minio.apm-minio-cluster.svc.cluster.local:80'
          access_key_id: NfrgmJdmBmKO1ncMp5G1f111
          secret_access_key: iftiYY3Q811Jbew17pxO11111pLd0cQVXXcCa8bkcG3sZP9O
          insecure:          true
    # Uncomment when using Grafana Enterprise Metrics
    # admin_client:
    #   storage:
    #     s3:
    #       bucket_name: my-admin-bucket
    #       access_key_id: ${AWS_ACCESS_KEY_ID}
    #       endpoint: s3.amazonaws.com
    #       secret_access_key: ${AWS_SECRET_ACCESS_KEY}
    alertmanager_storage:
      s3:
        bucket_name: mimir-alertmanager
    blocks_storage:
      backend: s3
      s3:
        bucket_name: mimir-k8s
    ruler_storage:
      s3:
        bucket_name:  mimir-ruler
    memberlist:
      cluster_label: mimir-k8s
    querier:
      max_concurrent: 128
    frontend:
      query_result_response_format: protobuf
      query_stats_enabled: false
      max_outstanding_per_tenant: 1000
      scheduler_worker_concurrency: 50
      cache_results: true
      parallelize_shardable_queries: true
    limits:
      native_histograms_ingestion_enabled: true
      max_global_series_per_user: 0
      max_label_name_length: 102400
      max_label_value_length: 102400
      max_label_names_per_series: 100
      ingestion_rate: 10000000000
      out_of_order_time_window: 5m
      ruler_max_rule_groups_per_tenant: 0
      max_fetched_chunks_per_query: 0
      max_query_parallelism: 240
    ingester:
      ring:
        replication_factor: 3
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    distributor:
      ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
      remote_timeout: 30s
    query_scheduler:
      max_outstanding_requests_per_tenant: 1600
      ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    ruler:
      query_frontend:
        address: dns:///mimir-query-frontend.monitoring.svc.cluster.local:9095
      rule_path: /data/
      ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    alertmanager:
      data_dir: /data/
      sharding_ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    compactor:
      compaction_interval: 10m
      data_dir: /data/
      deletion_delay: 1h
      first_level_compaction_wait_period: 25m
      max_closing_blocks_concurrency: 2
      max_opening_blocks_concurrency: 4
      sharding_ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    store_gateway:
      sharding_ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379


metaMonitoring:
  serviceMonitor:
    enabled: true
    namespace: monitoring
    labels:
      release: prometheus

My current configuration may not be optimal, which is likely causing high system latency. Can you give me some advice? Thank you very much.

0 replies

clcc2019 · 2024-09-04T03:10:32Z

clcc2019
Sep 4, 2024
Author

You can use version 2.10.5, the delay can be reduced to less than 100ms. After I upgraded to 2.13, the delay increased several times

activity_tracker:
  filepath: /data/metrics-activity.log
blocks_storage:
  backend: s3
  bucket_store:
    # ignore_blocks_within: 0s
    sync_interval: 3m0s
    block_sync_concurrency: 60
    tenant_sync_concurrency: 4
    meta_sync_concurrency: 60
    max_concurrent: 300
    chunks_cache:
      attributes_ttl: 24h
      # max_get_range_requests: 6
      backend: memcached
      memcached:
        addresses: dns+memcached.mimir.svc.cluster.local:11211
        max_idle_connections: 500
        connect_timeout: 8s
        timeout: 3s
        max_item_size: 10485760
    index_cache:
      backend: memcached
      memcached:
        addresses: dns+memcached.mimir.svc.cluster.local:11211
        max_idle_connections: 500
        connect_timeout: 8s
        timeout: 3s
        max_item_size: 10485760
    metadata_cache:
      backend: memcached
      memcached:
        addresses: dns+memcached.mimir.svc.cluster.local:11211
        max_idle_connections: 500
        connect_timeout: 8s
        timeout: 3s
        max_item_size: 10485760
    sync_dir: /data/tsdb-sync
  s3:
    ***
  tsdb:
    dir: /data/tsdb
    flush_blocks_on_shutdown: false
    # retention_period: 9h
    wal_compression_enabled: true
    wal_segment_size_bytes: 536870912
    # head_compaction_interval: 20
compactor:
  compaction_interval: 15m
  compaction_concurrency: 4
  max_compaction_time: 2h
  data_dir: /data
  deletion_delay: 1h
  max_closing_blocks_concurrency: 4
  max_opening_blocks_concurrency: 4
  max_block_upload_validation_concurrency: 4
  sharding_ring:
    wait_stability_min_duration: 1m
distributor:
  remote_timeout: 6s
  max_recv_msg_size: 1073741824
  reusable_ingester_push_workers: 4000
  # direct_otlp_translation_enabled: true
  retry_after_header:
    enabled: true
    base_seconds: 5
    max_backoff_exponent: 20
  ha_tracker:
    enable_ha_tracker: true
    kvstore:
      store: inmemory
  instance_limits:
    max_inflight_push_requests: 1500
    max_inflight_push_requests_bytes: 3145728000
    max_ingestion_rate: 700000
frontend: 
  grpc_client_config:
    grpc_compression: ""
    max_recv_msg_size: 8589934592
    max_send_msg_size: 8589934592
  log_queries_longer_than: 10s
  # scheduler_worker_concurrency: 30
  parallelize_shardable_queries: true
  cache_results: true
  results_cache:
    backend: memcached
    memcached:
      addresses: dns+memcached.mimir.svc.cluster.local:11211
      max_idle_connections: 500
      connect_timeout: 8s
      timeout: 3s
      max_item_size: 10485760
  scheduler_address: mimir-query-scheduler-headless.mimir.svc:9095
frontend_worker:
  # response_streaming_enabled: true
  grpc_client_config:
    grpc_compression: ""
    max_recv_msg_size: 8589934592
    max_send_msg_size: 8589934592
  scheduler_address: mimir-query-scheduler-headless.mimir.svc:9095
ingester:
  instance_limits:
    max_inflight_push_requests: 3000
  ring:
    final_sleep: 0s
    kvstore:
      store: memberlist
    min_ready_duration: 30s
    num_tokens: 512
    tokens_file_path: /data/tokens
ingester_client:
  grpc_client_config:
    grpc_compression: ""
    max_recv_msg_size: 8589934592
    max_send_msg_size: 8589934592
limits:
  accept_ha_samples: true
  compactor_block_upload_enabled: true
  #compactor_blocks_retention_period: 15d
  ha_cluster_label: cluster
  ha_max_clusters: 1000
  ha_replica_label: prometheus_replica
  ingestion_burst_size: 4000000
  ingestion_rate: 5000000
  results_cache_ttl: 24h
  compactor_split_and_merge_shards: 16
  compactor_split_groups: 4
  cache_unaligned_requests: false
  # results_cache_ttl_for_out_of_order_time_window: 30m
  label_values_max_cardinality_label_names_per_request: 300
  # max_fetched_series_per_query: 250000
  max_cache_freshness: 10m
  max_global_series_per_user: 0
  query_sharding_total_shards: 16
  # max_label_names_per_series: 600
  # max_metadata_length: 102400
  max_query_parallelism: 30
  #max_total_query_length: 15d
  # out_of_order_time_window: 2h
memberlist:
  abort_if_cluster_join_fails: false
  compression_enabled: false
  join_members:
  - dns+mimir-gossip-ring.mimir.svc.cluster.local:7946
multitenancy_enabled: false
querier:
  max_concurrent: 128
  # query_store_after: 8h
  # query_ingesters_within: 9h
query_scheduler:
  max_outstanding_requests_per_tenant: 3200
runtime_config:
  file: /var/mimir/runtime.yaml
server:
  grpc_server_max_concurrent_streams: 4000
  grpc_server_max_connection_age: 10m
  grpc_server_max_connection_age_grace: 10m
  grpc_server_max_connection_idle: 10m
  grpc_server_keepalive_time: 2h
  grpc_server_num_workers: 2000
  log_format: json
  log_level: info
store_gateway:
  sharding_ring:
    tokens_file_path: /data/tokens
    wait_stability_min_duration: 1m
    zone_awareness_enabled: false

1 reply

wangjinxiang0522 Sep 10, 2024

Yes, in version 2.13, I've encountered this issue as well. At the same time, I found that, under the condition of keeping the total resources constant, increasing the data of the ingester helps to reduce latency

wangjinxiang0522 · 2024-09-10T09:00:40Z

wangjinxiang0522
Sep 10, 2024

In the Kubernetes (K8s) environment, the version of Mimir being used is 2.11.0. （Version 2.13 has higher latency.）

Here's my current configuration:

image:
  repository: reg.sprucetec.com/monitor/grafana/mimir
  tag: r281-93e069f
  pullPolicy: IfNotPresent

global:
  dnsService: "kube-dns"
  # extraEnvFrom:
  #   - secretRef:
  #       name: mimir-bucket-secret
  podAnnotations:
    bucketSecretVersion: "0"

# This turns of the built-in MinIO support
minio:
  enabled: false
nginx:
  enabled: false


gateway:
  enabledNonEnterprise: true
  nodeSelector:
    node-type: monitoring
  replicas: 3
  nginxConfig:
    accessLogEnabled: false
  nginx:
    config:
      enableIPv6: false
    verboseLogging: false
    image:
      registry: reg.sprucetec.com
      repository: monitor/docker.io/nginxinc/nginx-unprivileged
      tag: 1.25-alpine
      pullPolicy: IfNotPresent

  service:
    legacyPorts: null
  resources:
    limits:
      memory: "4Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "2"



ingester:
  replicas: 80
  nodeSelector:
    node-type: monitoring
  zoneAwareReplication:
    enabled: false
  env:
    - name: GOMEMLIMIT
      value: "14745MiB"
    - name: GOMAXPROCS
      value: "4"
  resources:
    limits:
      memory: "16Gi"
      cpu: "4"
    requests:
      memory: "5Gi"
      cpu: "500m"
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"



distributor:
  replicas: 25
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "8Gi"
      cpu: "4"
    requests:
      memory: "256Mi"
      cpu: "200m"
  env:
    - name: GOMEMLIMIT
      value: "7372MiB"
    - name: GOMAXPROCS
      value: "4"



querier:
  replicas: 10
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "8Gi"
      cpu: "4"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "7372MiB"
    - name: GOMAXPROCS
      value: "4"


query_frontend:
  replicas: 4
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "100Mi"
      cpu: "200m"
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "4"



ruler:
  enabled: true
  replicas: 4
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "200Mi"
      cpu: "200m"
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "4"

alertmanager:
  nodeSelector:
    node-type: monitoring
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"

  replicas: 2
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "1Gi"
      cpu: "800m"
  statefulSet:
    enabled: true
  env:
    - name: GOMEMLIMIT
      value: "3686MiB"
    - name: GOMAXPROCS
      value: "4"

memcachedExporter:
  enabled: true
  image:
    repository: reg.sprucetec.com/monitor/memcached-exporter
    tag: v0.14.3
    pullPolicy: IfNotPresent

memcached:
  image:
    repository: reg.sprucetec.com/monitor/memcached
    tag: 1.6.25-alpine
    pullPolicy: IfNotPresent

admin-cache:
  enabled: true
  replicas: 1
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "16Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "14745MiB"
    - name: GOMAXPROCS
      value: "2"


chunks-cache:
  enabled: true
  replicas: 2
  allocatedMemory: 16384
  maxItemMemory: 10
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "18Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "16588MiB"
    - name: GOMAXPROCS
      value: "2"


index-cache:
  enabled: true
  replicas: 1
  allocatedMemory: 8192
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "10Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "8892MiB"
    - name: GOMAXPROCS
      value: "2"




metadata-cache:
  enabled: true
  nodeSelector:
    node-type: monitoring
  replicas: 1
  allocatedMemory: 1024
  maxItemMemory: 10
  resources:
    limits:
      memory: "2Gi"
      cpu: "1"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "1800MiB"
    - name: GOMAXPROCS
      value: "1"


results-cache:
  nodeSelector:
    node-type: monitoring
  enabled: true
  replicas: 2
  allocatedMemory: 16384
  maxItemMemory: 10
  resources:
    limits:
      memory: "18Gi"
      cpu: "2"
    requests:
      memory: "128Mi"
      cpu: "100m"
  env:
    - name: GOMEMLIMIT
      value: "16588MiB"
    - name: GOMAXPROCS
      value: "2"


store_gateway:
  replicas: 6
  zoneAwareReplication:
    enabled: false
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "16Gi"
      cpu: "6"
    requests:
      memory: "500Mi"
      cpu: "200m"
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"


compactor:
  replicas: 2
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "4"
    requests:
      memory: "1Gi"
      cpu: "100m"
  persistentVolume:
    enabled: true
    size: 60Gi
    storageClass: "csi-rbd-sc"
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "4"


overrides_exporter:
  enabled: true
  replicas: 2
  nodeSelector:
    node-type: monitoring
  resources:
    limits:
      memory: "4Gi"
      cpu: "1"
    requests:
      cpu: 50m
      memory: 64Mi
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "1"



query_scheduler:
  enabled: true
  replicas: 2
  nodeSelector:
    node-type: monitoring
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      memory: "4Gi"
      cpu: "2"
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "2"



rollout_operator:
  enabled: true
  image:
    repository: reg.sprucetec.com/monitor/grafana/rollout-operator
    tag: v0.13.0
    pullPolicy: IfNotPresent

  nodeSelector:
    node-type: monitoring
  resources:
    requests:
      cpu: 100m
      memory: 64Mi
    limits:
      memory: "4Gi"
      cpu: "2"
  env:
    - name: GOMEMLIMIT
      value: "3684MiB"
    - name: GOMAXPROCS
      value: "2"


mimir:
  structuredConfig:
    multitenancy_enabled: false
    server:
      grpc_server_max_recv_msg_size: 104857600
      grpc_server_max_send_msg_size: 104857600
      grpc_server_max_concurrent_streams: 1500
      log_level: warn
    common:
      storage:
        backend: s3
        s3:
          endpoint: 'minio.apm-minio-cluster.svc.cluster.local:80'
          access_key_id: NfrgmJdmB3mKO1n4cM4p5G1f111
          secret_access_key: iftiYY3Q8311Jb4ew17p4xO11111pLd0cQVXXcCa8bkcG3sZP9O
          insecure:          true
    # Uncomment when using Grafana Enterprise Metrics
    # admin_client:
    #   storage:
    #     s3:
    #       bucket_name: my-admin-bucket
    #       access_key_id: ${AWS_ACCESS_KEY_ID}
    #       endpoint: s3.amazonaws.com
    #       secret_access_key: ${AWS_SECRET_ACCESS_KEY}
    alertmanager_storage:
      s3:
        bucket_name: mimir-alertmanager
    blocks_storage:
      backend: s3
      s3:
        bucket_name: mimir-k8s
    ruler_storage:
      s3:
        bucket_name:  mimir-ruler
    memberlist:
      cluster_label: mimir-k8s
    querier:
      max_concurrent: 128
    frontend:
      query_result_response_format: protobuf
      query_stats_enabled: false
      max_outstanding_per_tenant: 1000
      scheduler_worker_concurrency: 50
      cache_results: true
      parallelize_shardable_queries: true
    limits:
      native_histograms_ingestion_enabled: true
      max_global_series_per_user: 0
      max_label_name_length: 102400
      max_label_value_length: 102400
      max_label_names_per_series: 100
      ingestion_rate: 10000000000
      out_of_order_time_window: 5m
      ruler_max_rule_groups_per_tenant: 0
      max_fetched_chunks_per_query: 0
      max_query_parallelism: 240
    ingester:
      ring:
        replication_factor: 3
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    distributor:
      ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
      remote_timeout: 30s
    query_scheduler:
      max_outstanding_requests_per_tenant: 1600
      ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    ruler:
      query_frontend:
        address: dns:///mimir-query-frontend.monitoring.svc.cluster.local:9095
      rule_path: /data/
      ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    alertmanager:
      data_dir: /data/
      sharding_ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    compactor:
      compaction_interval: 10m
      data_dir: /data/
      deletion_delay: 1h
      first_level_compaction_wait_period: 25m
      max_closing_blocks_concurrency: 2
      max_opening_blocks_concurrency: 4
      sharding_ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379
    store_gateway:
      sharding_ring:
        kvstore:
          store: etcd
          etcd:
            endpoints:
              - http://10.9.183.114:2379
              - http://10.9.182.106:2379
              - http://10.9.183.115:2379


metaMonitoring:
  serviceMonitor:
    enabled: true
    namespace: monitoring
    labels:
      release: prometheus

The Ingester is using 80 pods here, with each pod utilizing approximately 2.5 CPU cores and 4 GB of memory.

I feel that the Ingester is using a very high amount of resources. Could you provide information on the resource usage of your various components?

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Sudden high latency #9128

{{title}}

Replies: 3 comments 1 reply

{{title}}

{{title}}

{{title}}

{{title}}

{{editor}}'s edit

{{editor}}'s edit

Select a reply

Sudden high latency #9128

clcc2019 Aug 29, 2024

Replies: 3 comments · 1 reply

wangjinxiang0522 Aug 30, 2024

clcc2019 Sep 4, 2024 Author

wangjinxiang0522 Sep 10, 2024

wangjinxiang0522 Sep 10, 2024

clcc2019
Aug 29, 2024

Replies: 3 comments 1 reply

wangjinxiang0522
Aug 30, 2024

clcc2019
Sep 4, 2024
Author

wangjinxiang0522
Sep 10, 2024