From 0583144386fe07892bcb12a4a2dcd6e863a07bb8 Mon Sep 17 00:00:00 2001 From: amanmallsops Date: Thu, 25 Jan 2024 15:33:01 +0530 Subject: [PATCH 1/5] adding resource limit and request on slow log --- examples/complete/aws/helm/values.yaml | 108 ++++++++++++++++++------- helm/values/mysqldb/values.yaml | 14 ++++ 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/examples/complete/aws/helm/values.yaml b/examples/complete/aws/helm/values.yaml index 3d01d42..df132c2 100644 --- a/examples/complete/aws/helm/values.yaml +++ b/examples/complete/aws/helm/values.yaml @@ -1,35 +1,81 @@ primary: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "Infra-Services" - operator: In - values: - - "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Infra-Services" + operator: In + values: + - "true" + resources: + limits: + cpu: 350m + memory: 1Gi + requests: + cpu: 200m + memory: 700Mi + + sidecars: + - name: slow-log + image: busybox:1.28 + args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log'] + volumeMounts: + - name: data + mountPath: /bitnami/mysql + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 55m + memory: 156Mi secondary: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "Infra-Services" - operator: In - values: - - "true" - -affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "Infra-Services" - operator: In - values: - - "true" -backupjob: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Infra-Services" + operator: In + values: + - "true" + + resources: + limits: + cpu: 350m + memory: 1Gi + requests: + cpu: 200m + memory: 700Mi + + sidecars: + - name: slow-log + image: busybox:1.28 + args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log'] + volumeMounts: + - name: data + mountPath: /bitnami/mysql + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 55m + memory: 156Mi + +metrics: + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 10m + memory: 50Mi + + +backup: resources: requests: memory: 100Mi @@ -38,11 +84,11 @@ backupjob: memory: 200Mi cpu: 100m -restorejob: +restore: resources: requests: memory: 100Mi cpu: 50m limits: memory: 200Mi - cpu: 100m + cpu: 100m \ No newline at end of file diff --git a/helm/values/mysqldb/values.yaml b/helm/values/mysqldb/values.yaml index ed71018..3760675 100644 --- a/helm/values/mysqldb/values.yaml +++ b/helm/values/mysqldb/values.yaml @@ -467,6 +467,13 @@ primary: volumeMounts: - name: data mountPath: /bitnami/mysql + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 50m + memory: 128Mi ## MySQL Primary Service parameters @@ -843,6 +850,13 @@ secondary: volumeMounts: - name: data mountPath: /bitnami/mysql + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 50m + memory: 128Mi ## MySQL Secondary Service parameters ## service: From df58d2871ff5315fb0874a7af5f47760ef3e5cba Mon Sep 17 00:00:00 2001 From: amanmallsops Date: Thu, 25 Jan 2024 17:42:03 +0530 Subject: [PATCH 2/5] changes in helm value in example --- examples/complete/aws/helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/complete/aws/helm/values.yaml b/examples/complete/aws/helm/values.yaml index df132c2..3216b91 100644 --- a/examples/complete/aws/helm/values.yaml +++ b/examples/complete/aws/helm/values.yaml @@ -75,7 +75,7 @@ metrics: memory: 50Mi -backup: +backupjob: resources: requests: memory: 100Mi @@ -84,7 +84,7 @@ backup: memory: 200Mi cpu: 100m -restore: +restorejob: resources: requests: memory: 100Mi From 5e1ac678d8015cd63c39f402ae3cb41598f9a0ea Mon Sep 17 00:00:00 2001 From: amanmallsops Date: Sun, 28 Jan 2024 18:34:01 +0530 Subject: [PATCH 3/5] done require changes for resource limit and request --- examples/complete/aws/helm/values.yaml | 8 ++++---- main.tf | 6 ++++-- modules/backup/templates/cronjob.yaml | 3 ++- modules/restore/templates/job.yaml | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/examples/complete/aws/helm/values.yaml b/examples/complete/aws/helm/values.yaml index 3216b91..3d9fe0e 100644 --- a/examples/complete/aws/helm/values.yaml +++ b/examples/complete/aws/helm/values.yaml @@ -28,8 +28,8 @@ primary: cpu: 100m memory: 256Mi requests: - cpu: 55m - memory: 156Mi + cpu: 50m + memory: 128Mi secondary: affinity: @@ -62,8 +62,8 @@ secondary: cpu: 100m memory: 256Mi requests: - cpu: 55m - memory: 156Mi + cpu: 50m + memory: 128Mi metrics: resources: diff --git a/main.tf b/main.tf index 05121c5..0c96065 100644 --- a/main.tf +++ b/main.tf @@ -54,7 +54,8 @@ resource "helm_release" "mysqldb_backup" { azure_storage_account_key = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : "" azure_container_name = var.bucket_provider_type == "azure" ? var.azure_container_name : "" annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${var.iam_role_arn_backup}" : "iam.gke.io/gcp-service-account: ${var.service_account_backup}" - }) + }), + var.mysqldb_config.values_yaml ] } @@ -78,6 +79,7 @@ resource "helm_release" "mysqldb_restore" { azure_storage_account_key = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : "" azure_container_name = var.bucket_provider_type == "azure" ? var.azure_container_name : "" annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${var.iam_role_arn_restore}" : "iam.gke.io/gcp-service-account: ${var.service_account_restore}" - }) + }), + var.mysqldb_config.values_yaml ] } diff --git a/modules/backup/templates/cronjob.yaml b/modules/backup/templates/cronjob.yaml index 85d8cf2..431e688 100644 --- a/modules/backup/templates/cronjob.yaml +++ b/modules/backup/templates/cronjob.yaml @@ -43,4 +43,5 @@ spec: value: "s3" - name: AWS_DEFAULT_REGION value: {{ .Values.backup.aws_default_region }} - resources: {{ .Values.backupjob.resources | toYaml | nindent 12 }} + resources: + {{- toYaml .Values.backupjob.resources | nindent 14 }} diff --git a/modules/restore/templates/job.yaml b/modules/restore/templates/job.yaml index 8087a06..40d60f4 100644 --- a/modules/restore/templates/job.yaml +++ b/modules/restore/templates/job.yaml @@ -32,6 +32,7 @@ spec: value: {{ .Values.bucket_provider_type}} - name: AWS_DEFAULT_REGION value: {{ .Values.restore.aws_default_region}} - resources: {{ .Values.restorejob.resources | toYaml | nindent 12 }} + resources: + {{- toYaml .Values.restorejob.resources | nindent 14 }} restartPolicy: Never backoffLimit: 4 From 5a4a6d1ba1a06a8d3a075bfe8c927c3044ddecd7 Mon Sep 17 00:00:00 2001 From: amanmallsops Date: Tue, 30 Jan 2024 16:43:28 +0530 Subject: [PATCH 4/5] required changes implemented --- README.md | 9 ++ examples/complete/aws/helm/values.yaml | 149 +++++++++++++------------ helm/values/mysqldb/values.yaml | 32 ++++-- modules/backup/templates/cronjob.yaml | 3 +- modules/restore/templates/job.yaml | 3 +- 5 files changed, 112 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 4db01f4..c546f7c 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,15 @@ module "mysql" { ## IAM Permissions The required IAM permissions to create resources from this module can be found [here](https://github.com/squareops/terraform-kubernetes-mysql/blob/main/IAM.md) +## Backup +- In order to enable backup, it require database names like "db1, db2" or if it is blank it will backup all database without sys, information schema, performance schema and mysql. +- command using to do backup: +``` +mysqldump -h$HOST -u$USER -p$PASSWORD --databases db_name > full-backup.sql +``` +## Restore +- In order to enable backup, backup should be in .sql or .zip extention. + ## Important Notes 1. In order to enable the exporter, it is required to deploy Prometheus/Grafana first. 2. The exporter is a tool that extracts metrics data from an application or system and makes it available to be scraped by Prometheus. diff --git a/examples/complete/aws/helm/values.yaml b/examples/complete/aws/helm/values.yaml index 3d9fe0e..10c3425 100644 --- a/examples/complete/aws/helm/values.yaml +++ b/examples/complete/aws/helm/values.yaml @@ -1,94 +1,103 @@ primary: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "Infra-Services" - operator: In - values: - - "true" - resources: - limits: - cpu: 350m - memory: 1Gi - requests: - cpu: 200m - memory: 700Mi - - sidecars: - - name: slow-log - image: busybox:1.28 - args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log'] - volumeMounts: - - name: data - mountPath: /bitnami/mysql - resources: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Infra-Services" + operator: In + values: + - "true" + resources: limits: - cpu: 100m - memory: 256Mi + cpu: 350m + memory: 1Gi requests: - cpu: 50m - memory: 128Mi + cpu: 200m + memory: 700Mi -secondary: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "Infra-Services" - operator: In - values: - - "true" + sidecars: + - name: slow-log + image: busybox:1.28 + args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log'] + volumeMounts: + - name: data + mountPath: /bitnami/mysql + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 50m + memory: 128Mi - resources: - limits: - cpu: 350m - memory: 1Gi - requests: - cpu: 200m - memory: 700Mi +secondary: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Infra-Services" + operator: In + values: + - "true" - sidecars: - - name: slow-log - image: busybox:1.28 - args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log'] - volumeMounts: - - name: data - mountPath: /bitnami/mysql - resources: + resources: limits: - cpu: 100m - memory: 256Mi + cpu: 350m + memory: 1Gi requests: - cpu: 50m - memory: 128Mi + cpu: 200m + memory: 700Mi + + sidecars: + - name: slow-log + image: busybox:1.28 + args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log'] + volumeMounts: + - name: data + mountPath: /bitnami/mysql + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 50m + memory: 128Mi metrics: resources: limits: - cpu: 200m - memory: 500Mi + cpu: 200m + memory: 500Mi requests: - cpu: 10m - memory: 50Mi + cpu: 10m + memory: 50Mi +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Infra-Services" + operator: In + values: + - "true" backupjob: resources: requests: - memory: 100Mi - cpu: 50m - limits: - memory: 200Mi + memory: 250Mi cpu: 100m + limits: + memory: 500Mi + cpu: 200m restorejob: resources: requests: - memory: 100Mi - cpu: 50m + memory: 250Mi + cpu: 100m limits: - memory: 200Mi - cpu: 100m \ No newline at end of file + memory: 500Mi + cpu: 200m \ No newline at end of file diff --git a/helm/values/mysqldb/values.yaml b/helm/values/mysqldb/values.yaml index 3760675..a0817c4 100644 --- a/helm/values/mysqldb/values.yaml +++ b/helm/values/mysqldb/values.yaml @@ -468,12 +468,16 @@ primary: - name: data mountPath: /bitnami/mysql resources: - limits: - cpu: 100m - memory: 256Mi - requests: - cpu: 50m - memory: 128Mi + ## Example: + ## limits: + ## cpu: 100m + ## memory: 256Mi + limits: {} + ## Examples: + ## requests: + ## cpu: 100m + ## memory: 256Mi + requests: {} ## MySQL Primary Service parameters @@ -851,12 +855,16 @@ secondary: - name: data mountPath: /bitnami/mysql resources: - limits: - cpu: 100m - memory: 256Mi - requests: - cpu: 50m - memory: 128Mi + ## Example: + ## limits: + ## cpu: 100m + ## memory: 256Mi + limits: {} + ## Examples: + ## requests: + ## cpu: 100m + ## memory: 256Mi + requests: {} ## MySQL Secondary Service parameters ## service: diff --git a/modules/backup/templates/cronjob.yaml b/modules/backup/templates/cronjob.yaml index 431e688..4a94b24 100644 --- a/modules/backup/templates/cronjob.yaml +++ b/modules/backup/templates/cronjob.yaml @@ -13,7 +13,8 @@ spec: spec: template: spec: - affinity: {{ .Values.affinity | toYaml | nindent 10 }} + affinity: + {{- toYaml .Values.affinity | nindent 12 }} restartPolicy: OnFailure imagePullSecrets: - name: regcred diff --git a/modules/restore/templates/job.yaml b/modules/restore/templates/job.yaml index 40d60f4..49fc9ca 100644 --- a/modules/restore/templates/job.yaml +++ b/modules/restore/templates/job.yaml @@ -5,7 +5,8 @@ metadata: spec: template: spec: - affinity: {{ .Values.affinity | toYaml | nindent 6 }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} serviceAccountName: sa-mysql-restore containers: - name: restore-mysqldb From 1309be932edea084fbc76d2df5cb69402e334d40 Mon Sep 17 00:00:00 2001 From: amanmallsops Date: Tue, 30 Jan 2024 17:23:14 +0530 Subject: [PATCH 5/5] added detail for backup and restore readme, done changes in values.yaml of example and of module of backup and restore and done changes in template of backup nad restore --- README.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c546f7c..1bec542 100644 --- a/README.md +++ b/README.md @@ -116,15 +116,32 @@ module "mysql" { ## IAM Permissions The required IAM permissions to create resources from this module can be found [here](https://github.com/squareops/terraform-kubernetes-mysql/blob/main/IAM.md) -## Backup -- In order to enable backup, it require database names like "db1, db2" or if it is blank it will backup all database without sys, information schema, performance schema and mysql. +## MySQL Backup and Restore +This module provides functionality to automate the backup and restore process for MySQL databases using AWS S3 buckets. It allows users to easily schedule backups, restore databases from backups stored in S3, and manage access permissions using AWS IAM roles. +Features +### Backup +- Users can schedule full backups. +- upports specifying individual database names for backup or backing up all databases except system databases. +- Backups are stored in specified S3 buckets. +### Restore +- Users can restore MySQL databases from backups stored in S3 buckets. +- Supports specifying the backup file to restore from and the target S3 bucket region. +### IAM Role for Permissions +- Users need to provide an IAM role for the module to access the specified S3 bucket and perform backup and restore operations. +## Module Inputs +### Backup Configuration - command using to do backup: ``` mysqldump -h$HOST -u$USER -p$PASSWORD --databases db_name > full-backup.sql ``` -## Restore -- In order to enable backup, backup should be in .sql or .zip extention. - +- mysql_database_name: The name of the MySQL database to backup. Leave blank to backup all databases except system databases. +- bucket_uri: The URI of the S3 bucket where backups will be stored. +- s3_bucket_region: The region of the S3 bucket. +- cron_for_full_backup: The cron expression for scheduling full backups. +### Restore Configuration +- mysqldb_restore_config: Configuration for restoring databases.bucket_uri: The URI of the S3 bucket containing the backup file. +- file_name: The name of the backup file to restore. +- s3_bucket_region: The region of the S3 bucket containing the backup file. ## Important Notes 1. In order to enable the exporter, it is required to deploy Prometheus/Grafana first. 2. The exporter is a tool that extracts metrics data from an application or system and makes it available to be scraped by Prometheus.