Merge pull request #17 from amanmallsops/feature/resource-limit-request

feature adding resource limit and request on slow log

Author: RohitSquareops

README.md

@@ -116,6 +116,32 @@ module "mysql" {
 ## IAM Permissions
 The required IAM permissions to create resources from this module can be found [here](https://github.com/squareops/terraform-kubernetes-mysql/blob/main/IAM.md)
 
+## MySQL Backup and Restore
+This module provides functionality to automate the backup and restore process for MySQL databases using AWS S3 buckets. It allows users to easily schedule backups, restore databases from backups stored in S3, and manage access permissions using AWS IAM roles.
+Features
+### Backup
+- Users can schedule full backups.
+- upports specifying individual database names for backup or backing up all databases except system databases.
+- Backups are stored in specified S3 buckets.
+### Restore
+- Users can restore MySQL databases from backups stored in S3 buckets.
+- Supports specifying the backup file to restore from and the target S3 bucket region.
+### IAM Role for Permissions
+- Users need to provide an IAM role for the module to access the specified S3 bucket and perform backup and restore operations.
+## Module Inputs
+### Backup Configuration
+- command using to do backup:
+```
+mysqldump -h$HOST -u$USER -p$PASSWORD --databases db_name > full-backup.sql
+```
+- mysql_database_name: The name of the MySQL database to backup. Leave blank to backup all databases except system databases.
+- bucket_uri: The URI of the S3 bucket where backups will be stored.
+- s3_bucket_region: The region of the S3 bucket.
+- cron_for_full_backup: The cron expression for scheduling full backups.
+### Restore Configuration
+- mysqldb_restore_config: Configuration for restoring databases.bucket_uri: The URI of the S3 bucket containing the backup file.
+- file_name: The name of the backup file to restore.
+- s3_bucket_region: The region of the S3 bucket containing the backup file.
 ## Important Notes
   1. In order to enable the exporter, it is required to deploy Prometheus/Grafana first.
   2. The exporter is a tool that extracts metrics data from an application or system and makes it available to be scraped by Prometheus.

examples/complete/aws/helm/values.yaml

@@ -8,6 +8,28 @@ primary:
                       operator: In
                       values:
                       - "true"
+    resources:
+        limits:
+            cpu: 350m
+            memory: 1Gi
+        requests:
+            cpu: 200m
+            memory: 700Mi
+
+    sidecars:
+        - name: slow-log
+          image: busybox:1.28
+          args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log']
+          volumeMounts:
+              - name: data
+                mountPath: /bitnami/mysql
+          resources:
+              limits:
+                cpu: 100m
+                memory: 256Mi
+              requests:
+                cpu: 50m
+                memory: 128Mi
 
 secondary:
     affinity:
@@ -20,29 +42,62 @@ secondary:
                       values:
                       - "true"
 
+    resources:
+        limits:
+            cpu: 350m
+            memory: 1Gi
+        requests:
+            cpu: 200m
+            memory: 700Mi
+
+    sidecars:
+        - name: slow-log
+          image: busybox:1.28
+          args: [/bin/sh, -c, 'tail -n+1 -F /bitnami/mysql/slow-log.log']
+          volumeMounts:
+              - name: data
+                mountPath: /bitnami/mysql
+          resources:
+              limits:
+                cpu: 100m
+                memory: 256Mi
+              requests:
+                cpu: 50m
+                memory: 128Mi
+
+metrics:
+  resources:
+    limits:
+      cpu: 200m
+      memory: 500Mi
+    requests:
+      cpu: 10m
+      memory: 50Mi
+
+
 affinity:
-  nodeAffinity:
-    requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "Infra-Services"
-            operator: In
-            values:
-            - "true"
+    nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+                - key: "Infra-Services"
+                  operator: In
+                  values:
+                  - "true"
 backupjob:
   resources:
     requests:
-      memory: 100Mi
-      cpu: 50m
-    limits:
-      memory: 200Mi
+      memory: 250Mi
       cpu: 100m
+    limits:
+      memory: 500Mi
+      cpu: 200m
 
 restorejob:
   resources:
     requests:
-      memory: 100Mi
-      cpu: 50m
-    limits:
-      memory: 200Mi
+      memory: 250Mi
       cpu: 100m
+    limits:
+      memory: 500Mi
+      cpu: 200m

helm/values/mysqldb/values.yaml

@@ -467,6 +467,17 @@ primary:
       volumeMounts:
       - name: data
         mountPath: /bitnami/mysql
+      resources:
+        ## Example:
+        ## limits:
+        ##    cpu: 100m
+        ##    memory: 256Mi
+        limits: {}
+        ## Examples:
+        ## requests:
+        ##    cpu: 100m
+        ##    memory: 256Mi
+        requests: {}
 
 
   ## MySQL Primary Service parameters
@@ -843,6 +854,17 @@ secondary:
       volumeMounts:
       - name: data
         mountPath: /bitnami/mysql
+      resources:
+        ## Example:
+        ## limits:
+        ##    cpu: 100m
+        ##    memory: 256Mi
+        limits: {}
+        ## Examples:
+        ## requests:
+        ##    cpu: 100m
+        ##    memory: 256Mi
+        requests: {}
   ## MySQL Secondary Service parameters
   ##
   service:

main.tf

@@ -54,7 +54,8 @@ resource "helm_release" "mysqldb_backup" {
       azure_storage_account_key  = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : ""
       azure_container_name       = var.bucket_provider_type == "azure" ? var.azure_container_name : ""
       annotations                = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${var.iam_role_arn_backup}" : "iam.gke.io/gcp-service-account: ${var.service_account_backup}"
-    })
+    }),
+    var.mysqldb_config.values_yaml
   ]
 }
 
@@ -78,6 +79,7 @@ resource "helm_release" "mysqldb_restore" {
       azure_storage_account_key  = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : ""
       azure_container_name       = var.bucket_provider_type == "azure" ? var.azure_container_name : ""
       annotations                = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${var.iam_role_arn_restore}" : "iam.gke.io/gcp-service-account: ${var.service_account_restore}"
-    })
+    }),
+    var.mysqldb_config.values_yaml
   ]
 }

modules/backup/templates/cronjob.yaml

@@ -13,7 +13,8 @@ spec:
     spec:
       template:
         spec:
-          affinity: {{ .Values.affinity | toYaml | nindent 10 }}
+          affinity: 
+            {{- toYaml .Values.affinity | nindent 12 }}
           restartPolicy: OnFailure
           imagePullSecrets:
           - name: regcred
@@ -43,4 +44,5 @@ spec:
               value:  "s3"
             - name: AWS_DEFAULT_REGION
               value: {{ .Values.backup.aws_default_region }}
-            resources: {{ .Values.backupjob.resources | toYaml | nindent 12 }}
+            resources: 
+              {{- toYaml .Values.backupjob.resources | nindent 14 }}

modules/restore/templates/job.yaml

@@ -5,7 +5,8 @@ metadata:
 spec:
   template:
     spec:
-      affinity: {{ .Values.affinity | toYaml | nindent 6 }}
+      affinity:
+        {{- toYaml .Values.affinity | nindent 8 }}
       serviceAccountName: sa-mysql-restore
       containers:
           - name: restore-mysqldb
@@ -32,6 +33,7 @@ spec:
               value: {{ .Values.bucket_provider_type}}
             - name: AWS_DEFAULT_REGION
               value: {{ .Values.restore.aws_default_region}}
-            resources: {{ .Values.restorejob.resources | toYaml | nindent 12 }}
+            resources:
+              {{- toYaml .Values.restorejob.resources | nindent 14 }}
       restartPolicy: Never
   backoffLimit: 4