diff --git a/production/ksonnet/loki/boltdb_shipper.libsonnet b/production/ksonnet/loki/boltdb_shipper.libsonnet index 6ba06004a82b..a257230d845e 100644 --- a/production/ksonnet/loki/boltdb_shipper.libsonnet +++ b/production/ksonnet/loki/boltdb_shipper.libsonnet @@ -40,6 +40,11 @@ 'boltdb.shipper.cache-location': '/data/boltdb-cache', } else {}, + ruler_args+:: if $._config.using_boltdb_shipper then { + // Use PVC for caching + 'boltdb.shipper.cache-location': '/data/boltdb-cache', + } else {}, + // we don't dedupe index writes when using boltdb-shipper so don't deploy a cache for it. memcached_index_writes: if $._config.using_boltdb_shipper then {} else super.memcached_index_writes, diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index 8253b68e5c68..3e64175a0dff 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -23,6 +23,10 @@ querier_pvc_size: '10Gi', querier_pvc_class: 'fast', + stateful_rulers: false, + ruler_pvc_size: '10Gi', + ruler_pvc_class: 'fast', + compactor_pvc_size: '10Gi', compactor_pvc_class: 'fast', diff --git a/production/ksonnet/loki/ruler.libsonnet b/production/ksonnet/loki/ruler.libsonnet index f769d2fd09bb..23298ab8121f 100644 --- a/production/ksonnet/loki/ruler.libsonnet +++ b/production/ksonnet/loki/ruler.libsonnet @@ -1,5 +1,9 @@ { local container = $.core.v1.container, + local pvc = $.core.v1.persistentVolumeClaim, + local statefulSet = $.apps.v1.statefulSet, + local deployment = $.apps.v1.deployment, + local volumeMount = $.core.v1.volumeMount, ruler_args:: $._config.commonArgs { target: 'ruler', @@ -13,13 +17,15 @@ $.util.resourcesRequests('1', '6Gi') + $.util.resourcesLimits('16', '16Gi') + $.util.readinessProbe + - $.jaeger_mixin + $.jaeger_mixin + + if $._config.stateful_rulers then + container.withVolumeMountsMixin([ + volumeMount.new('ruler-data', '/data'), + ]) else {} else {}, - local deployment = $.apps.v1.deployment, - ruler_deployment: - if $._config.ruler_enabled then + if $._config.ruler_enabled && !$._config.stateful_rulers then deployment.new('ruler', 2, [$.ruler_container]) + deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(600) + $.config_hash_mixin + @@ -28,10 +34,31 @@ $.util.antiAffinity else {}, - local service = $.core.v1.service, + ruler_service: if !$._config.ruler_enabled + then {} + else + if $._config.stateful_rulers + then $.util.serviceFor($.ruler_statefulset) + else $.util.serviceFor($.ruler_deployment), - ruler_service: - if $._config.ruler_enabled then - $.util.serviceFor($.ruler_deployment) - else {}, + + // PVC for rulers when running as statefulsets + ruler_data_pvc:: if $._config.ruler_enabled && $._config.stateful_rulers then + pvc.new('ruler-data') + + pvc.mixin.spec.resources.withRequests({ storage: $._config.ruler_pvc_size }) + + pvc.mixin.spec.withAccessModes(['ReadWriteOnce']) + + pvc.mixin.spec.withStorageClassName($._config.ruler_pvc_class) + else {}, + + ruler_statefulset: if $._config.ruler_enabled && $._config.stateful_rulers then + statefulSet.new('ruler', 2, [$.ruler_container], $.ruler_data_pvc) + + statefulSet.mixin.spec.withServiceName('ruler') + + statefulSet.mixin.spec.withPodManagementPolicy('Parallel') + + $.config_hash_mixin + + $.util.configVolumeMount('loki', '/etc/loki/config') + + $.util.configVolumeMount('overrides', '/etc/loki/overrides') + + $.util.antiAffinity + + statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + + statefulSet.mixin.spec.template.spec.securityContext.withFsGroup(10001) // 10001 is the group ID assigned to Loki in the Dockerfile + else {}, }