From dfe2372aab8a5ed6a5e507a98917ffd68c6162d0 Mon Sep 17 00:00:00 2001 From: Janis Bebritis Date: Tue, 26 Mar 2024 09:43:59 +0200 Subject: [PATCH 1/2] elasticdump for elasticsearch data backup --- charts/drupal/templates/_helpers.tpl | 53 ++++++++++++++++++++++++++-- charts/drupal/values.schema.json | 2 ++ charts/drupal/values.yaml | 8 +++-- silta/php.Dockerfile | 5 +++ silta/shell.Dockerfile | 5 +++ silta/silta.yml | 13 +++++++ 6 files changed, 81 insertions(+), 5 deletions(-) diff --git a/charts/drupal/templates/_helpers.tpl b/charts/drupal/templates/_helpers.tpl index c619a11db..e135cc507 100644 --- a/charts/drupal/templates/_helpers.tpl +++ b/charts/drupal/templates/_helpers.tpl @@ -369,9 +369,18 @@ done {{- end }} {{- end }} - {{ if .Values.elasticsearch.enabled }} - {{ include "drupal.wait-for-elasticsearch-command" . }} - {{ end }} + {{- if .Values.elasticsearch.enabled }} + {{- include "drupal.wait-for-elasticsearch-command" . }} + {{- if and .Release.IsInstall .Values.referenceData.elasticsearch }} + if [[ -f /app/reference-data/elasticsearch.tar.gz || -f /app/reference-data/elasticsearch.tar.gz ]]; then + echo "Importing elasticsearch reference data" + mkdir /tmp/elasticsearch + tar -xzf /app/reference-data/elasticsearch.tar.gz -C /tmp/elasticsearch + multielasticdump --direction=load --input=/tmp/elasticsearch --output=http://${ELASTICSEARCH_HOST}:9200/ + echo "Elasticsearch reference data imported" + fi + {{- end }} + {{- end }} {{ if .Release.IsInstall }} {{ .Values.php.postinstall.command }} @@ -395,6 +404,7 @@ done {{- define "drupal.extract-reference-data" -}} set -e if [[ "$(drush status --fields=bootstrap)" = *'Successful'* ]] ; then + echo "Dump reference database." dump_dir=/tmp/reference-data-export/ mkdir "${dump_dir}" @@ -462,6 +472,15 @@ if [[ "$(drush status --fields=bootstrap)" = *'Successful'* ]] ; then /app/reference-data/{{ $index }} {{ end -}} {{- end }} + + {{- if .Values.referenceData.elasticsearch }} + echo "Creating reference data for elasticsearch" + mkdir /tmp/elasticsearch + multielasticdump --direction=dump --input=http://${ELASTICSEARCH_HOST}:9200/ --output=/tmp/elasticsearch + tar -czf /tmp/elasticsearch.tar.gz -C /tmp/elasticsearch . + cp /tmp/elasticsearch.tar.gz /app/reference-data + {{- end }} + else echo "Drupal is not installed, skipping reference database dump." fi @@ -519,6 +538,9 @@ fi {{- define "drupal.backup-command" -}} {{ include "drupal.backup-command.dump-database" . }} + {{- if and .Values.elasticsearch.enabled .Values.backup.elasticsearch }} + {{- include "drupal.backup-command.dump-elasticsearch" . }} + {{- end }} {{ include "drupal.backup-command.archive-store-backup" . }} {{- end }} @@ -545,6 +567,24 @@ fi echo "Database backup complete." {{- end }} +{{- define "drupal.backup-command.dump-elasticsearch" }} + set -e + + if ! command -v multielasticdump &> /dev/null + then + echo "multielasticdump could not be found. Elasticsearch backup failed." + exit 1 + fi + + # Dump elasticsearch data + echo "Starting Elasticsearch backup." + mkdir /tmp/elasticsearch + multielasticdump --direction=dump --input=http://${ELASTICSEARCH_HOST}:9200/ --output=/tmp/elasticsearch + tar -czf /tmp/elasticsearch.tar.gz -C /tmp/elasticsearch . + + echo "Elasticsearch backup complete." +{{- end }} + {{- define "drupal.backup-command.archive-store-backup" -}} # Compress the database dump and copy it into the backup folder. @@ -554,6 +594,13 @@ fi # Create a folder for the backup mkdir -p $BACKUP_LOCATION + + {{- if .Values.elasticsearch.enabled }} + # Copy the elasticsearch dump into the backup folder. + cp /tmp/elasticsearch.tar.gz $BACKUP_LOCATION/ + {{- end }} + + # Copy the database dump into the backup folder. cp /tmp/db.sql.gz $BACKUP_LOCATION/db.sql.gz {{- if not .Values.backup.skipFiles }} diff --git a/charts/drupal/values.schema.json b/charts/drupal/values.schema.json index 48511ada8..784953168 100644 --- a/charts/drupal/values.schema.json +++ b/charts/drupal/values.schema.json @@ -448,6 +448,7 @@ "storageClassName": { "type": "string" }, "csiDriverName": { "type": "string" }, "skipMount": { "type": "boolean" }, + "elasticsearch": { "type": "boolean" }, "resources": { "type": "object", "additionalProperties": false, @@ -488,6 +489,7 @@ "csiDriverName": { "type": "string" }, "ignoreTableContent": { "type": "string" }, "skipFiles": { "type": "boolean" }, + "elasticsearch": { "type": "boolean" }, "resources": { "type": "object", "additionalProperties": false, diff --git a/charts/drupal/values.yaml b/charts/drupal/values.yaml index a1b11abc7..a4ad61501 100644 --- a/charts/drupal/values.yaml +++ b/charts/drupal/values.yaml @@ -450,6 +450,8 @@ referenceData: - languages - php + elasticsearch: true + # Files larger than this will not be included in reference data. maxFileSize: '5M' @@ -530,8 +532,10 @@ backup: # These tables will have their content ignored from the backups. ignoreTableContent: '(cache|cache_.*)' - # Do not backup files - #skipFiles: true + # Do not backup files + skipFiles: false + + elasticsearch: true # Resources for the backup cron job. resources: diff --git a/silta/php.Dockerfile b/silta/php.Dockerfile index 61e1d5fa3..7cf87b50f 100644 --- a/silta/php.Dockerfile +++ b/silta/php.Dockerfile @@ -3,4 +3,9 @@ FROM wunderio/silta-php-fpm:8.2-fpm-v1 COPY --chown=www-data:www-data . /app +##### +# Install Elasticdump +RUN apk add npm nodejs && npm install -g elasticdump +##### + USER www-data diff --git a/silta/shell.Dockerfile b/silta/shell.Dockerfile index 5c7dc7b6e..7c824afd5 100644 --- a/silta/shell.Dockerfile +++ b/silta/shell.Dockerfile @@ -1,4 +1,9 @@ # Dockerfile for the Drupal container. FROM wunderio/silta-php-shell:php8.2-v1 +##### +# Install Elasticdump (skip when php base image contains it) +RUN apk add npm nodejs && npm install -g elasticdump +##### + COPY --chown=www-data:www-data . /app diff --git a/silta/silta.yml b/silta/silta.yml index 8feafebe3..76334ca90 100644 --- a/silta/silta.yml +++ b/silta/silta.yml @@ -55,3 +55,16 @@ php: # The ~ symbol will be replaced by a random digit from 0 to 9. # This will avoid running all cron jobs at the same time. schedule: '~ 0 31 2 *' + +backup: + enabled: true + schedule: "*/10 * * * *" + retention: 3 + +referenceData: + enabled: true + referenceEnvironment: 'feature/esdump' + +# TODO: wait for https://github.com/wunderio/silta-images/pull/180 +# TODO: remove elasticdump installation from local dockerfiles +# TODO: remove test values From 8b3e5c7c9f6b475a7c39f7fe52f55bf7c822b570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=81nis=20Bebr=C4=ABtis?= Date: Tue, 3 Sep 2024 14:06:47 +0300 Subject: [PATCH 2/2] Removing test values --- silta/php.Dockerfile | 5 ----- silta/shell.Dockerfile | 5 ----- silta/silta.yml | 13 ------------- 3 files changed, 23 deletions(-) diff --git a/silta/php.Dockerfile b/silta/php.Dockerfile index 7cf87b50f..61e1d5fa3 100644 --- a/silta/php.Dockerfile +++ b/silta/php.Dockerfile @@ -3,9 +3,4 @@ FROM wunderio/silta-php-fpm:8.2-fpm-v1 COPY --chown=www-data:www-data . /app -##### -# Install Elasticdump -RUN apk add npm nodejs && npm install -g elasticdump -##### - USER www-data diff --git a/silta/shell.Dockerfile b/silta/shell.Dockerfile index 7c824afd5..5c7dc7b6e 100644 --- a/silta/shell.Dockerfile +++ b/silta/shell.Dockerfile @@ -1,9 +1,4 @@ # Dockerfile for the Drupal container. FROM wunderio/silta-php-shell:php8.2-v1 -##### -# Install Elasticdump (skip when php base image contains it) -RUN apk add npm nodejs && npm install -g elasticdump -##### - COPY --chown=www-data:www-data . /app diff --git a/silta/silta.yml b/silta/silta.yml index 76334ca90..8feafebe3 100644 --- a/silta/silta.yml +++ b/silta/silta.yml @@ -55,16 +55,3 @@ php: # The ~ symbol will be replaced by a random digit from 0 to 9. # This will avoid running all cron jobs at the same time. schedule: '~ 0 31 2 *' - -backup: - enabled: true - schedule: "*/10 * * * *" - retention: 3 - -referenceData: - enabled: true - referenceEnvironment: 'feature/esdump' - -# TODO: wait for https://github.com/wunderio/silta-images/pull/180 -# TODO: remove elasticdump installation from local dockerfiles -# TODO: remove test values