From b892e391548b95a4cc54d4fda22b3e4075ae34a7 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 00:19:29 -0400 Subject: [PATCH 01/15] Initial, limited, snapshot ability Adds Make snapshot-image target to publish a local/idc-snapshot:latest image. Notably, the snapshots cannot be pushed or pulled from docker container registries yet --- .env | 2 +- .gitignore | 1 + docker-compose.idc-snapshot.yml | 23 +++++++++++++++++++++++ idc.Makefile | 11 +++++++++++ snapshot/Dockerfile | 2 ++ snapshot/snapshot.Dockerfile | 6 ++++++ 6 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 docker-compose.idc-snapshot.yml create mode 100644 snapshot/Dockerfile create mode 100644 snapshot/snapshot.Dockerfile diff --git a/.env b/.env index 29f934f18..9a72fb7b4 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ # See documentation for more details. ENVIRONMENT=local -REQUIRED_SERIVCES=activemq alpaca cantaloupe crayfish crayfits drupal mariadb matomo solr +REQUIRED_SERIVCES=activemq alpaca cantaloupe crayfish crayfits drupal mariadb matomo solr idc-snapshot ############################################################################### # Environment variables specific to composer. ############################################################################### diff --git a/.gitignore b/.gitignore index 26347b845..3d1898cd1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ codebase/.gitattributes codebase/web/core/ codebase/web/modules/contrib/ codebase/web/themes/contrib/ +snapshot/data.tar # ========================= # Packages diff --git a/docker-compose.idc-snapshot.yml b/docker-compose.idc-snapshot.yml new file mode 100644 index 000000000..d0d541bc7 --- /dev/null +++ b/docker-compose.idc-snapshot.yml @@ -0,0 +1,23 @@ +services: + drupal: + depends_on: + - snapshot + mariadb: + depends_on: + - snapshot + matomo: + depends_on: + - snapshot + solr: + depends_on: + - snapshot + snapshot: + container_name: snapshot + build: ./snapshot + image: idc/snapshot + volumes: + - drupal-sites-data:/data/drupal + - mariadb-data:/data/mariadb-data + - mariadb-files:/data/mariadb-files + - matomo-config-data:/data/matomo-config + - solr-data:/data/solr diff --git a/idc.Makefile b/idc.Makefile index d25483068..ae330e113 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -23,3 +23,14 @@ destroy-state: composer-install: echo "Installing via composer" docker-compose exec drupal with-contenv bash -lc 'COMPOSER_MEMORY_LIMIT=-1 composer install' + +.PHONY: snapshot-image +.SILENT: snapshot-image +snapshot-image: + docker-compose stop + docker run --rm --volumes-from snapshot \ + -v ${PWD}/snapshot:/dump \ + alpine:latest \ + /bin/tar cvf /dump/data.tar /data + docker build -f snapshot/snapshot.Dockerfile -t local/idc-snapshot:latest ./snapshot + docker-compose up -d \ No newline at end of file diff --git a/snapshot/Dockerfile b/snapshot/Dockerfile new file mode 100644 index 000000000..162153742 --- /dev/null +++ b/snapshot/Dockerfile @@ -0,0 +1,2 @@ +FROM alpine:latest +COPY data/ / diff --git a/snapshot/snapshot.Dockerfile b/snapshot/snapshot.Dockerfile new file mode 100644 index 000000000..50feba2da --- /dev/null +++ b/snapshot/snapshot.Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:latest +COPY data.tar / +RUN tar xvf /data.tar -C / + +FROM alpine:latest +COPY --from=0 /data/ /data From ee42b6e4630ab99720e88ed245f05df7e0b06c18 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 00:29:11 -0400 Subject: [PATCH 02/15] Move cache rebuild to its own Make target --- idc.Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/idc.Makefile b/idc.Makefile index ae330e113..9468762df 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -7,8 +7,14 @@ .PHONY: bootstrap .SILENT: bootstrap bootstrap: default destroy-state composer-install install \ - update-settings-php update-config-from-environment solr-cores run-islandora-migrations - echo "ebuilding Drupal cache..." + update-settings-php update-config-from-environment solr-cores run-islandora-migrations \ + cache-rebuild + +# Rebuilds the Drupal cache +.PHONY: cache-rebuild +.SILENT: cache-rebuild +cache-rebuild: + echo "irebuilding Drupal cache..." docker-compose exec drupal drush cr -y .PHONY: destroy-state From 49209a22e6b8c077109736ea1c6cfadaabfd25ab Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 00:30:56 -0400 Subject: [PATCH 03/15] Create snapshot image as part of bootstrap --- idc.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idc.Makefile b/idc.Makefile index 9468762df..0f151558c 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -8,7 +8,7 @@ .SILENT: bootstrap bootstrap: default destroy-state composer-install install \ update-settings-php update-config-from-environment solr-cores run-islandora-migrations \ - cache-rebuild + cache-rebuild snapshot-image # Rebuilds the Drupal cache .PHONY: cache-rebuild From a409e4d819fd926368f5a9c071514e529209daa1 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 04:14:18 -0400 Subject: [PATCH 04/15] Tag images appropriately when making snapshot Uses variables from .env for naming snapshot image Whe `make snapshot-image` is invoked, the snapshot image will be given a unique tag name, .env will be updated, and the docker-compowe.yml file will be re-generated so that it contains the new tag. That way, a developer can choose to commit the new snapshot they built easily (by committing .env), and a docker-compose down -v will result in restarting to the state of their last snapshot. --- .env | 3 +++ docker-compose.idc-snapshot.yml | 2 +- idc.Makefile | 7 +++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 9a72fb7b4..1e6c59dbf 100644 --- a/.env +++ b/.env @@ -62,3 +62,6 @@ REPOSITORY=islandora # their versions specified explicitly in their respective docker-compose files. TAG=latest +# Docker image and tag for snapshot image +SNAPSHOT_IMAGE=birkland/snapshot +SNAPSHOT_TAG=upstream-20201007-739693ae-11-g49209a22.1602143780 diff --git a/docker-compose.idc-snapshot.yml b/docker-compose.idc-snapshot.yml index d0d541bc7..24f3dfa89 100644 --- a/docker-compose.idc-snapshot.yml +++ b/docker-compose.idc-snapshot.yml @@ -14,7 +14,7 @@ services: snapshot: container_name: snapshot build: ./snapshot - image: idc/snapshot + image: ${SNAPSHOT_IMAGE}:${SNAPSHOT_TAG} volumes: - drupal-sites-data:/data/drupal - mariadb-data:/data/mariadb-data diff --git a/idc.Makefile b/idc.Makefile index 0f151558c..cecb66798 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -1,4 +1,3 @@ - # Bootstrap a new instance without Fedora. Assumes there is a Drupal site in ./codebase. # Will do a clean Drupal install and initialization # @@ -38,5 +37,9 @@ snapshot-image: -v ${PWD}/snapshot:/dump \ alpine:latest \ /bin/tar cvf /dump/data.tar /data - docker build -f snapshot/snapshot.Dockerfile -t local/idc-snapshot:latest ./snapshot + TAG=`git describe --tags`.`date +%s` && \ + docker build -f snapshot/snapshot.Dockerfile -t ${SNAPSHOT_IMAGE}:$$TAG ./snapshot && \ + sed -i s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=$$TAG/ .env + rm docker-compose.yml + $(MAKE) docker-compose.yml docker-compose up -d \ No newline at end of file From 3a5ea25c2ddee434fece2c593801ffb10c312e34 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 04:44:37 -0400 Subject: [PATCH 05/15] Make sure bootstrap builds against empty snapshot --- .env | 2 +- idc.Makefile | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 1e6c59dbf..e5e789ad3 100644 --- a/.env +++ b/.env @@ -64,4 +64,4 @@ TAG=latest # Docker image and tag for snapshot image SNAPSHOT_IMAGE=birkland/snapshot -SNAPSHOT_TAG=upstream-20201007-739693ae-11-g49209a22.1602143780 +SNAPSHOT_TAG=upstream-20201007-739693ae-12-ga409e4d8.1602146397 diff --git a/idc.Makefile b/idc.Makefile index cecb66798..facfb393f 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -5,7 +5,7 @@ # otherwise we could of simply done 'hydrate' instead of update-settings-php, update-config... etc) .PHONY: bootstrap .SILENT: bootstrap -bootstrap: default destroy-state composer-install install \ +bootstrap: snapshot-empty destroy-state default composer-install install \ update-settings-php update-config-from-environment solr-cores run-islandora-migrations \ cache-rebuild snapshot-image @@ -13,7 +13,7 @@ bootstrap: default destroy-state composer-install install \ .PHONY: cache-rebuild .SILENT: cache-rebuild cache-rebuild: - echo "irebuilding Drupal cache..." + echo "rebuilding Drupal cache..." docker-compose exec drupal drush cr -y .PHONY: destroy-state @@ -42,4 +42,12 @@ snapshot-image: sed -i s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=$$TAG/ .env rm docker-compose.yml $(MAKE) docker-compose.yml - docker-compose up -d \ No newline at end of file + docker-compose up -d + +.PHONY: snapshot-empty +.SILENT: snapshot-empty +snapshot-empty: + rm docker-compose.yml + sed -i s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=empty/ .env + $(MAKE) docker-compose.yml + docker-compose build snapshot From a927d2c11de18860a97ffcd7905a425c86b4f505 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 18:31:03 -0400 Subject: [PATCH 06/15] make sure default is default Make target --- idc.Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/idc.Makefile b/idc.Makefile index facfb393f..811c2d2f9 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -1,3 +1,5 @@ +.DEFAULT_GOAL := default + # Bootstrap a new instance without Fedora. Assumes there is a Drupal site in ./codebase. # Will do a clean Drupal install and initialization # From 3a6c710c82bfe46a18194bfc072ab7249cd39dfb Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 19:14:23 -0400 Subject: [PATCH 07/15] Add 'up' make target to start up dev environment runs composer install as part of bringing up the IDC stack --- idc.Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/idc.Makefile b/idc.Makefile index 811c2d2f9..23c7012d4 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -53,3 +53,13 @@ snapshot-empty: sed -i s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=empty/ .env $(MAKE) docker-compose.yml docker-compose build snapshot + +.PHONY: up +.SILENT: up +up: default start composer-install + + +.PHONY: start +.SILENT: start +start: + docker-compose up -d From 03ebf8e805b8327d6cd6860cbebc326f1d94b834 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 20:22:05 -0400 Subject: [PATCH 08/15] Update README --- IDC.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/IDC.md b/IDC.md index b575c3e71..03e62609e 100644 --- a/IDC.md +++ b/IDC.md @@ -3,15 +3,81 @@ Leverages ISLE to provide a local development environment for the IDC stack, with particular focus on development/testing of the Drupal site. -## Quick Start - -Right now, this is in an incomplete state. You can build an initial Drupal site from scratch via executing - - make bootstrap - ## Contents * Our Drupal site is in `codebase`. * Use `composer` to add, remove, or update dependencies in `codebase/composer.json` and `codebase/composer.lock` when developing * Dependencies are not vendored, so you need to do a composer install. This is included in `make bootstrap` * IDC development-specific environment variables are in `.env` and `docker-compose.env.yml` +* An idc-specific Makefile `idc.Makefile` defines additional make targets available for `make + +## Quick Start + +To start the IDC development environment, run + + make up + +This will build a `docker-compose` file, run `composer install` to locally install all dependencies for our site (which will +take a few minutes when done the first time, but will be much quicker subsequent times), and start the stack. The stack will +start from a known snapshot state, which currently is an entirely empty (but initialized) Drupal. + +To reset to a known Drupal state, run + + docker-compose down -v + docker-compose up -d + +This will remove all content from volumes that you may have added, launch using the snapshot as its initial state. + +To dump the site's configuration so that it can be committed to `git`, do + + make config-export + +To take a snapshot of Drupal's current content, do + + make snapshot + +See [snapshots](#snapshots) for more information on how to make and publish snapshots + +## Snapshots + +Snapshots are Docker images that contain Drupal state (content files, database, SOLR indexes, Fedora files, etc). When Docker starts, +all Docker volumes will be populated with files from the snapshot image. The net result is that an environment will start quickly, +from a known state, with pre-populated content. + +After Docker starts from a snapshot, data subsequent in Docker's volumes is ephemeral. It will persist across `stop` and `down`, but can be wiped out by + + docker-compose down -v + +When docker subsequently starts, it will start from the known snapshot state. You are free to [take a snapshot](#taking-and-publishing-snapshots) +whenever you want a checkpoint you can reliably reset Drupal to. + +### Images +The image used for the snapshot is specified via environment variables in `.env`. For example: + + # Docker image and tag for snapshot image + SNAPSHOT_IMAGE=birkland/snapshot + SNAPSHOT_TAG=upstream-20201007-739693ae-12-ga409e4d8.1602146397 + +When the `docker-compose.yml` make target is run, that image and tag will be specified in the docker-compose file. The images contain data that are +copied to Docker volumes upon initial startup of the stack (i.e. snapshots are deployed only once, until all volumes are wiped out via `docker-compose down -v`). Because they are just regular docker images, they can be pushed and puled from container registry as usual. + +### Taking and publishing snapshots +To take a snapshot, run + + make snapshot + +This will do the following: +* stop the docker-compose stack +* dump the contents of the volumes +* create a new image from the contents of the volume +* give the image a unique tag based on the current git commit, and the date +* update the `.env` file to specify the just-taken `SNAPSHOT_TAG` +* rebuild the `docker-compose.yml` file to specify that tag +* start docker-compose + +If you want to commit that snapshot so that others can use it, you need to commit `.env` (which contains the tag of the snapshot image), +and publish the snapshot image to a Docker registry via + + docker-compose push snapshot + +Make sure you do both steps! You need to push the image (so others can pull it), and push `.env` (so others can check out and run it). \ No newline at end of file From a74be01d3b7584004a1eeb581187ff3f1656947b Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 20:26:27 -0400 Subject: [PATCH 09/15] specifiy docker compose version --- docker-compose.idc-snapshot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.idc-snapshot.yml b/docker-compose.idc-snapshot.yml index 24f3dfa89..b993616ea 100644 --- a/docker-compose.idc-snapshot.yml +++ b/docker-compose.idc-snapshot.yml @@ -1,3 +1,4 @@ +version: "3.7" services: drupal: depends_on: From e99c912f6f5c0b74bde699843bad6c003e070338 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Thu, 8 Oct 2020 20:39:17 -0400 Subject: [PATCH 10/15] Add empty snapshot data --- .gitignore | 1 - snapshot/data/drupal/.keep | 0 snapshot/data/mariadb-data/.keep | 0 snapshot/data/mariadb-files/.keep | 0 snapshot/data/matomo-config/.keep | 0 snapshot/data/minio/.keep | 0 snapshot/data/solr/.keep | 0 7 files changed, 1 deletion(-) create mode 100644 snapshot/data/drupal/.keep create mode 100644 snapshot/data/mariadb-data/.keep create mode 100644 snapshot/data/mariadb-files/.keep create mode 100644 snapshot/data/matomo-config/.keep create mode 100644 snapshot/data/minio/.keep create mode 100644 snapshot/data/solr/.keep diff --git a/.gitignore b/.gitignore index 3d1898cd1..37be597f4 100644 --- a/.gitignore +++ b/.gitignore @@ -66,7 +66,6 @@ Thumbs.db #=================== # ISLE specific -data logs site .vagrant diff --git a/snapshot/data/drupal/.keep b/snapshot/data/drupal/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/snapshot/data/mariadb-data/.keep b/snapshot/data/mariadb-data/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/snapshot/data/mariadb-files/.keep b/snapshot/data/mariadb-files/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/snapshot/data/matomo-config/.keep b/snapshot/data/matomo-config/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/snapshot/data/minio/.keep b/snapshot/data/minio/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/snapshot/data/solr/.keep b/snapshot/data/solr/.keep new file mode 100644 index 000000000..e69de29bb From b20579b73525d7e9343228cb32e35d3d3d4701d3 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Fri, 9 Oct 2020 02:51:41 -0400 Subject: [PATCH 11/15] Do not pull images by default --- idc.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/idc.Makefile b/idc.Makefile index 23c7012d4..7254da338 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -7,7 +7,7 @@ # otherwise we could of simply done 'hydrate' instead of update-settings-php, update-config... etc) .PHONY: bootstrap .SILENT: bootstrap -bootstrap: snapshot-empty destroy-state default composer-install install \ +bootstrap: snapshot-empty destroy-state up install \ update-settings-php update-config-from-environment solr-cores run-islandora-migrations \ cache-rebuild snapshot-image @@ -56,7 +56,7 @@ snapshot-empty: .PHONY: up .SILENT: up -up: default start composer-install +up: download-default-certs docker-compose.yml start composer-install .PHONY: start From 27ba32ef648c6be7182e0f7132e774257d5a3f71 Mon Sep 17 00:00:00 2001 From: Elliot Metsger Date: Mon, 12 Oct 2020 09:45:15 -0400 Subject: [PATCH 12/15] Account for incompatibilities between gnu and bsd sed Allow the removal of docker-compose.yml to fail --- idc.Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/idc.Makefile b/idc.Makefile index 7254da338..4e5f570c3 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -41,7 +41,9 @@ snapshot-image: /bin/tar cvf /dump/data.tar /data TAG=`git describe --tags`.`date +%s` && \ docker build -f snapshot/snapshot.Dockerfile -t ${SNAPSHOT_IMAGE}:$$TAG ./snapshot && \ - sed -i s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=$$TAG/ .env + cat .env | sed s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=$$TAG/ > /tmp/.env && \ + cp /tmp/.env .env && \ + rm /tmp/.env rm docker-compose.yml $(MAKE) docker-compose.yml docker-compose up -d @@ -49,8 +51,10 @@ snapshot-image: .PHONY: snapshot-empty .SILENT: snapshot-empty snapshot-empty: - rm docker-compose.yml - sed -i s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=empty/ .env + -rm docker-compose.yml + sed s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=empty/ .env > /tmp/.env && \ + cp /tmp/.env .env && \ + rm /tmp/.env $(MAKE) docker-compose.yml docker-compose build snapshot From bb81c80f928cb0a44a57e39244bbb1c86bf7932a Mon Sep 17 00:00:00 2001 From: Elliot Metsger Date: Mon, 12 Oct 2020 09:46:39 -0400 Subject: [PATCH 13/15] Invoke default when making bootstrap. --- idc.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idc.Makefile b/idc.Makefile index 4e5f570c3..4397187c0 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -7,7 +7,7 @@ # otherwise we could of simply done 'hydrate' instead of update-settings-php, update-config... etc) .PHONY: bootstrap .SILENT: bootstrap -bootstrap: snapshot-empty destroy-state up install \ +bootstrap: default snapshot-empty destroy-state up install \ update-settings-php update-config-from-environment solr-cores run-islandora-migrations \ cache-rebuild snapshot-image From 4af37eb7d752d56ede64258701de3996117dcde5 Mon Sep 17 00:00:00 2001 From: Elliot Metsger Date: Mon, 12 Oct 2020 11:28:23 -0400 Subject: [PATCH 14/15] Bootstrapping is meant to affect the local development environment only, so revert any changes that the bootstrap target makes to .env. --- idc.Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idc.Makefile b/idc.Makefile index 4397187c0..2e40a9550 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -9,7 +9,8 @@ .SILENT: bootstrap bootstrap: default snapshot-empty destroy-state up install \ update-settings-php update-config-from-environment solr-cores run-islandora-migrations \ - cache-rebuild snapshot-image + cache-rebuild \ + git checkout -- .env # Rebuilds the Drupal cache .PHONY: cache-rebuild From 2354a9724566c4b3023460f39184638710d1e00b Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Tue, 13 Oct 2020 09:12:54 -0400 Subject: [PATCH 15/15] Fix typo --- idc.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idc.Makefile b/idc.Makefile index 2e40a9550..a12108189 100644 --- a/idc.Makefile +++ b/idc.Makefile @@ -9,7 +9,7 @@ .SILENT: bootstrap bootstrap: default snapshot-empty destroy-state up install \ update-settings-php update-config-from-environment solr-cores run-islandora-migrations \ - cache-rebuild \ + cache-rebuild git checkout -- .env # Rebuilds the Drupal cache