From c934b7ac6c3b5161db1afd83c5522ba736be9b93 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 8 Feb 2024 15:49:39 +0000 Subject: [PATCH 1/9] Increase the default PVC size for logstash --- docs/orchestrating-elastic-stack-applications/logstash.asciidoc | 2 +- pkg/controller/logstash/volume/defaults.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc index 39b5068140..c2568908bb 100644 --- a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc @@ -301,7 +301,7 @@ WARNING: Volume support for Logstash is a breaking change to earlier versions of [discrete] == Specifying the volume claim settings -By default, a PersistentVolume called `logstash-data` is created, that maps to `/usr/share/logstash/data` for persistent storage, typically used for storage from plugins. The `logstash-data` volume claim is, by default, a small (1Gi) volume, using the standard StorageClass of your Kubernetes cluster, but can be overridden by adding a `spec.volumeClaimTemplate` section named `logstash-data`. +By default, a PersistentVolume called `logstash-data` is created, that maps to `/usr/share/logstash/data` for persistent storage, typically used for storage from plugins. The `logstash-data` volume claim is, by default, a 1.5Gi volume, using the standard StorageClass of your Kubernetes cluster, but can be overridden by adding a `spec.volumeClaimTemplate` section named `logstash-data`. For production workloads, you should define your own volume claim template with the desired storage capacity and (optionally) the Kubernetes link:https://kubernetes.io/docs/concepts/storage/storage-classes/[storage class] to associate with the persistent volume. To override this volume claim for `data` usages, the name of this volume claim must be `logstash-data`. diff --git a/pkg/controller/logstash/volume/defaults.go b/pkg/controller/logstash/volume/defaults.go index 5890310ab4..fd9df54fc1 100644 --- a/pkg/controller/logstash/volume/defaults.go +++ b/pkg/controller/logstash/volume/defaults.go @@ -21,7 +21,7 @@ var ( ContainerMountPath: ConfigMountPath, } - DefaultPersistentVolumeSize = resource.MustParse("1Gi") + DefaultPersistentVolumeSize = resource.MustParse("1.5Gi") // DefaultDataVolumeClaim is the default data volume claim for Logstash pods. // We default to a 1GB persistent volume, using the default storage class. From f3d0c9c3adfede41b4371584932a3a5e6d72435f Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 8 Feb 2024 16:02:15 +0000 Subject: [PATCH 2/9] update unit tests --- .../logstash/logstash_controller_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/controller/logstash/logstash_controller_test.go b/pkg/controller/logstash/logstash_controller_test.go index ce50e7cf29..07da9d1ce5 100644 --- a/pkg/controller/logstash/logstash_controller_test.go +++ b/pkg/controller/logstash/logstash_controller_test.go @@ -191,7 +191,7 @@ func TestReconcileLogstash_Reconcile(t *testing.T) { StorageClassName: ptr.To[string](sampleStorageClass.Name), Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + corev1.ResourceStorage: resource.MustParse("1.5Gi"), }, }, }, @@ -303,7 +303,7 @@ func TestReconcileLogstash_Reconcile(t *testing.T) { StorageClassName: ptr.To[string](sampleStorageClass.Name), Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + corev1.ResourceStorage: resource.MustParse("1.5Gi"), }, }, }, @@ -432,7 +432,7 @@ func TestReconcileLogstash_Reconcile(t *testing.T) { StorageClassName: ptr.To[string](sampleStorageClass.Name), Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + corev1.ResourceStorage: resource.MustParse("1.5Gi"), }, }, }, @@ -550,7 +550,7 @@ func TestReconcileLogstash_Reconcile(t *testing.T) { StorageClassName: ptr.To[string](sampleStorageClass.Name), Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + corev1.ResourceStorage: resource.MustParse("1.5Gi"), }, }, }, @@ -650,7 +650,7 @@ func TestReconcileLogstash_Resize(t *testing.T) { }{ { name: "Cannot increase storage with fixed storage class", - initialCapacity: "1Gi", + initialCapacity: "1.5Gi", desiredCapacity: "3Gi", storageClass: fixedStorageClass, wantErr: true, @@ -658,7 +658,7 @@ func TestReconcileLogstash_Resize(t *testing.T) { { name: "Cannot decrease storage with resizable storage class", initialCapacity: "3Gi", - desiredCapacity: "1Gi", + desiredCapacity: "1.5Gi", storageClass: resizableStorageClass, wantErr: true, }, @@ -671,14 +671,14 @@ func TestReconcileLogstash_Resize(t *testing.T) { }, { name: "Nothing happens when keeping the storage the same with resizable storage class", - initialCapacity: "1Gi", - desiredCapacity: "1Gi", + initialCapacity: "1.5Gi", + desiredCapacity: "1.5Gi", storageClass: resizableStorageClass, wantErr: false, }, { name: "Can successfully resize the storage with resizable storage class", - initialCapacity: "1Gi", + initialCapacity: "1.5Gi", desiredCapacity: "3Gi", storageClass: resizableStorageClass, extraVerify: func(r ReconcileLogstash, desiredCapacity string) (reconcile.Result, error) { From 23f25aa24d384c562ffa3b26999939e30ee5e15e Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 8 Feb 2024 16:16:53 +0000 Subject: [PATCH 3/9] update doc Co-authored-by: Rob Bavey --- .../orchestrating-elastic-stack-applications/logstash.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc index c2568908bb..2eb20dd9d6 100644 --- a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc @@ -326,8 +326,9 @@ spec: storage: 2Gi ---- +In particular, when enabling the persistent queue (PQ) feature, using dead letter queues (DLQ), or certain plugins that make heavy use of temporary storage, the default volume size will likely be insufficient for production workloads and you should increase the storage capacity, or consider creating separate volumes for these use cases. -Separate storage, for example for Logstash configurations using persistent queues (PQ) and/or dead letter queues (DLQ), can be added by including an additional `spec.volumeClaimTemplate` along with a corresponding `spec.podTemplate.spec.containers.volumeMount` for each requested volume. +Separate storage can be added by including an additional `spec.volumeClaimTemplate` along with a corresponding `spec.podTemplate.spec.containers.volumeMount` for each requested volume. This example shows how to setup separate storage for a PQ: From 9bbf3fa295724ddcdc5fc5ae2278fa449b685b0b Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 8 Feb 2024 18:52:09 +0000 Subject: [PATCH 4/9] Update docs/orchestrating-elastic-stack-applications/logstash.asciidoc Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- .../logstash.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc index 2eb20dd9d6..d465b840dd 100644 --- a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc @@ -301,7 +301,11 @@ WARNING: Volume support for Logstash is a breaking change to earlier versions of [discrete] == Specifying the volume claim settings -By default, a PersistentVolume called `logstash-data` is created, that maps to `/usr/share/logstash/data` for persistent storage, typically used for storage from plugins. The `logstash-data` volume claim is, by default, a 1.5Gi volume, using the standard StorageClass of your Kubernetes cluster, but can be overridden by adding a `spec.volumeClaimTemplate` section named `logstash-data`. +A PersistentVolume called `logstash-data` is created by default. +It maps to `/usr/share/logstash/data` for persistent storage, which is typically used for storage from plugins. + +By default, the `logstash-data` volume claim is a 1.5Gi volume, using the standard StorageClass of your Kubernetes cluster. +You can override the default by adding a `spec.volumeClaimTemplate` section named `logstash-data`. For production workloads, you should define your own volume claim template with the desired storage capacity and (optionally) the Kubernetes link:https://kubernetes.io/docs/concepts/storage/storage-classes/[storage class] to associate with the persistent volume. To override this volume claim for `data` usages, the name of this volume claim must be `logstash-data`. From b1e09c9d89dd958ff4048526155380e6158bb017 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 8 Feb 2024 18:52:17 +0000 Subject: [PATCH 5/9] Update docs/orchestrating-elastic-stack-applications/logstash.asciidoc Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- .../logstash.asciidoc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc index d465b840dd..5c8b8d75f1 100644 --- a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc @@ -330,7 +330,13 @@ spec: storage: 2Gi ---- -In particular, when enabling the persistent queue (PQ) feature, using dead letter queues (DLQ), or certain plugins that make heavy use of temporary storage, the default volume size will likely be insufficient for production workloads and you should increase the storage capacity, or consider creating separate volumes for these use cases. +The default volume size will likely be insufficient for production workloads, espeically when you are using: + +* the persistent queue (PQ) feature +* dead letter queues (DLQ), or +* {ls} plugins that make heavy use of temporary storage. + +Increase the storage capacity, or consider creating separate volumes for these use cases. Separate storage can be added by including an additional `spec.volumeClaimTemplate` along with a corresponding `spec.podTemplate.spec.containers.volumeMount` for each requested volume. From 9caa23168e408facf09b1da7c475b01ea18ef616 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 8 Feb 2024 18:52:25 +0000 Subject: [PATCH 6/9] Update docs/orchestrating-elastic-stack-applications/logstash.asciidoc Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/orchestrating-elastic-stack-applications/logstash.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc index 5c8b8d75f1..536c6eaea0 100644 --- a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc @@ -338,7 +338,7 @@ The default volume size will likely be insufficient for production workloads, es Increase the storage capacity, or consider creating separate volumes for these use cases. -Separate storage can be added by including an additional `spec.volumeClaimTemplate` along with a corresponding `spec.podTemplate.spec.containers.volumeMount` for each requested volume. +You can add separate storage by including an additional `spec.volumeClaimTemplate` along with a corresponding `spec.podTemplate.spec.containers.volumeMount` for each requested volume. This example shows how to setup separate storage for a PQ: From 3536e39bca010178381db026d74758aef69f3dd9 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:29:04 +0000 Subject: [PATCH 7/9] Update docs/orchestrating-elastic-stack-applications/logstash.asciidoc Co-authored-by: Michael Morello --- docs/orchestrating-elastic-stack-applications/logstash.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc index 536c6eaea0..7834d7a5b6 100644 --- a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc @@ -330,7 +330,7 @@ spec: storage: 2Gi ---- -The default volume size will likely be insufficient for production workloads, espeically when you are using: +The default volume size will likely be insufficient for production workloads, especially when you are using: * the persistent queue (PQ) feature * dead letter queues (DLQ), or From b8671c0bf968476574e96dca13084e6a1ab25a57 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:29:25 +0000 Subject: [PATCH 8/9] Update docs/orchestrating-elastic-stack-applications/logstash.asciidoc Co-authored-by: Michael Morello --- docs/orchestrating-elastic-stack-applications/logstash.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc index 7834d7a5b6..9752f687b4 100644 --- a/docs/orchestrating-elastic-stack-applications/logstash.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/logstash.asciidoc @@ -304,7 +304,7 @@ WARNING: Volume support for Logstash is a breaking change to earlier versions of A PersistentVolume called `logstash-data` is created by default. It maps to `/usr/share/logstash/data` for persistent storage, which is typically used for storage from plugins. -By default, the `logstash-data` volume claim is a 1.5Gi volume, using the standard StorageClass of your Kubernetes cluster. +By default, the `logstash-data` volume claim is a `1.5Gi` volume, using the standard StorageClass of your Kubernetes cluster. You can override the default by adding a `spec.volumeClaimTemplate` section named `logstash-data`. For production workloads, you should define your own volume claim template with the desired storage capacity and (optionally) the Kubernetes link:https://kubernetes.io/docs/concepts/storage/storage-classes/[storage class] to associate with the persistent volume. To override this volume claim for `data` usages, the name of this volume claim must be `logstash-data`. From 43cd3e0c6e0f88a09ecbba638a5343e026b0333f Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:29:45 +0000 Subject: [PATCH 9/9] Update pkg/controller/logstash/volume/defaults.go Co-authored-by: Michael Morello --- pkg/controller/logstash/volume/defaults.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/logstash/volume/defaults.go b/pkg/controller/logstash/volume/defaults.go index fd9df54fc1..73a05baac5 100644 --- a/pkg/controller/logstash/volume/defaults.go +++ b/pkg/controller/logstash/volume/defaults.go @@ -24,7 +24,7 @@ var ( DefaultPersistentVolumeSize = resource.MustParse("1.5Gi") // DefaultDataVolumeClaim is the default data volume claim for Logstash pods. - // We default to a 1GB persistent volume, using the default storage class. + // We default to a 1.5Gi persistent volume, using the default storage class. DefaultDataVolumeClaim = corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: LogstashDataVolumeName,