Merge pull request #2 from birkland/snapshots

Updates to iDC-specific Make targets, including the support of snapshot image creation.

.env

@@ -10,7 +10,7 @@
 # See documentation for more details.
 ENVIRONMENT=local
 
-REQUIRED_SERIVCES=activemq alpaca cantaloupe crayfish crayfits drupal mariadb matomo solr
+REQUIRED_SERIVCES=activemq alpaca cantaloupe crayfish crayfits drupal mariadb matomo solr idc-snapshot
 ###############################################################################
 # Environment variables specific to composer.
 ###############################################################################
@@ -62,3 +62,6 @@ REPOSITORY=islandora
 # their versions specified explicitly in their respective docker-compose files.
 TAG=latest
 
+# Docker image and tag for snapshot image
+SNAPSHOT_IMAGE=birkland/snapshot
+SNAPSHOT_TAG=upstream-20201007-739693ae-12-ga409e4d8.1602146397

.gitignore

@@ -21,6 +21,7 @@ codebase/.gitattributes
 codebase/web/core/
 codebase/web/modules/contrib/
 codebase/web/themes/contrib/
+snapshot/data.tar
 
 # =========================
 # Packages
@@ -65,7 +66,6 @@ Thumbs.db
 
 #===================
 # ISLE specific
-data
 logs
 site
 .vagrant

IDC.md

@@ -3,15 +3,81 @@
 Leverages ISLE to provide a local development environment for the IDC stack, with particular focus on development/testing of 
 the Drupal site.
 
-## Quick Start
-
-Right now, this is in an incomplete state.  You can build an initial Drupal site from scratch via executing
-
-    make bootstrap
-
 ## Contents
 
 * Our Drupal site is in `codebase`.  
   * Use `composer` to add, remove, or update dependencies in `codebase/composer.json` and `codebase/composer.lock` when developing
   * Dependencies are not vendored, so you need to do a composer install.  This is included in `make bootstrap`
 * IDC development-specific environment variables are in `.env` and `docker-compose.env.yml`
+* An idc-specific Makefile `idc.Makefile` defines additional make targets available for `make
+
+## Quick Start
+
+To start the IDC development environment, run 
+
+    make up
+
+This will build a `docker-compose` file, run `composer install` to locally install all dependencies for our site (which will 
+take a few minutes when done the first time, but will be much quicker subsequent times), and start the stack.  The stack will
+start from a known snapshot state, which currently is an entirely empty (but initialized) Drupal. 
+
+To reset to a known Drupal state, run
+
+    docker-compose down -v
+    docker-compose up -d
+
+This will remove all content from volumes that you may have added, launch using the snapshot as its initial state.
+
+To dump the site's configuration so that it can be committed to `git`, do
+
+    make config-export
+
+To take a snapshot of Drupal's current content, do
+
+    make snapshot
+
+See [snapshots](#snapshots) for more information on how to make and publish snapshots
+
+## Snapshots
+
+Snapshots are Docker images that contain Drupal state (content files, database, SOLR indexes, Fedora files, etc).  When Docker starts,
+all Docker volumes will be populated with files from the snapshot image.  The net result is that an environment will start quickly,
+from a known state, with pre-populated content.
+
+After Docker starts from a snapshot, data subsequent in Docker's volumes is ephemeral.  It will persist across `stop` and `down`, but can be wiped out by
+
+    docker-compose down -v
+
+When docker subsequently starts, it will start from the known snapshot state.  You are free to [take a snapshot](#taking-and-publishing-snapshots)
+whenever you want a checkpoint you can reliably reset Drupal to.  
+
+### Images
+The image used for the snapshot is specified via environment variables in `.env`.  For example:
+
+    # Docker image and tag for snapshot image
+    SNAPSHOT_IMAGE=birkland/snapshot
+    SNAPSHOT_TAG=upstream-20201007-739693ae-12-ga409e4d8.1602146397
+
+When the `docker-compose.yml` make target is run, that image and tag will be specified in the docker-compose file.  The images contain data that are
+copied to Docker volumes upon initial startup of the stack (i.e. snapshots are deployed only once, until all volumes are wiped out via `docker-compose down -v`).  Because they are just regular docker images, they can be pushed and puled from container registry as usual.
+
+### Taking and publishing snapshots
+To take a snapshot, run 
+
+    make snapshot
+
+This will do the following:
+* stop the docker-compose stack
+* dump the contents of the volumes
+* create a new image from the contents of the volume
+* give the image a unique tag based on the current git commit, and the date
+* update the `.env` file to specify the just-taken `SNAPSHOT_TAG`
+* rebuild the `docker-compose.yml` file to specify that tag
+* start docker-compose
+
+If you want to commit that snapshot so that others can use it, you need to commit `.env` (which contains the tag of the snapshot image),
+and publish the snapshot image to a Docker registry via
+
+    docker-compose push snapshot
+
+Make sure you do both steps!  You need to push the image (so others can pull it), and push `.env` (so others can check out and run it).

docker-compose.idc-snapshot.yml

@@ -0,0 +1,24 @@
+version: "3.7"
+services:
+  drupal:
+    depends_on:
+    - snapshot
+  mariadb:
+    depends_on:
+    - snapshot
+  matomo:
+    depends_on:
+    - snapshot
+  solr:
+    depends_on:
+    - snapshot
+  snapshot:
+    container_name: snapshot
+    build: ./snapshot
+    image: ${SNAPSHOT_IMAGE}:${SNAPSHOT_TAG}
+    volumes:
+    - drupal-sites-data:/data/drupal
+    - mariadb-data:/data/mariadb-data
+    - mariadb-files:/data/mariadb-files
+    - matomo-config-data:/data/matomo-config
+    - solr-data:/data/solr

idc.Makefile

@@ -1,3 +1,4 @@
+.DEFAULT_GOAL := default
 
 # Bootstrap a new instance without Fedora.  Assumes there is a Drupal site in ./codebase.
 # Will do a clean Drupal install and initialization
@@ -6,9 +7,16 @@
 # otherwise we could of simply done 'hydrate' instead of update-settings-php, update-config... etc)
 .PHONY: bootstrap
 .SILENT: bootstrap
-bootstrap: default destroy-state composer-install install \
-		update-settings-php update-config-from-environment solr-cores run-islandora-migrations 
-	echo "ebuilding Drupal cache..."
+bootstrap: default snapshot-empty destroy-state up install \
+		update-settings-php update-config-from-environment solr-cores run-islandora-migrations \
+		cache-rebuild 
+		git checkout -- .env
+
+# Rebuilds the Drupal cache
+.PHONY: cache-rebuild
+.SILENT: cache-rebuild
+cache-rebuild:
+	echo "rebuilding Drupal cache..."
 	docker-compose exec drupal drush cr -y
 
 .PHONY: destroy-state
@@ -23,3 +31,40 @@ destroy-state:
 composer-install:
 	echo "Installing via composer"
 	docker-compose exec drupal with-contenv bash -lc 'COMPOSER_MEMORY_LIMIT=-1 composer install'
+
+.PHONY: snapshot-image
+.SILENT: snapshot-image
+snapshot-image:
+	docker-compose stop
+	docker run --rm --volumes-from snapshot \
+		-v ${PWD}/snapshot:/dump \
+		alpine:latest \
+		/bin/tar cvf /dump/data.tar /data
+	TAG=`git describe --tags`.`date +%s` && \
+		docker build -f snapshot/snapshot.Dockerfile -t ${SNAPSHOT_IMAGE}:$$TAG ./snapshot && \
+		cat .env | sed s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=$$TAG/ > /tmp/.env && \
+	  cp /tmp/.env .env && \
+	  rm /tmp/.env
+	rm docker-compose.yml
+	$(MAKE) docker-compose.yml
+	docker-compose up -d
+
+.PHONY: snapshot-empty
+.SILENT: snapshot-empty
+snapshot-empty:
+	-rm docker-compose.yml
+	sed s/SNAPSHOT_TAG=.*/SNAPSHOT_TAG=empty/ .env > /tmp/.env && \
+      cp /tmp/.env .env && \
+	    rm /tmp/.env
+	$(MAKE) docker-compose.yml
+	docker-compose build snapshot
+
+.PHONY: up
+.SILENT: up
+up:  download-default-certs docker-compose.yml start composer-install
+
+
+.PHONY: start
+.SILENT: start
+start:
+	docker-compose up -d

snapshot/Dockerfile

@@ -0,0 +1,2 @@
+FROM alpine:latest
+COPY data/ /

snapshot/snapshot.Dockerfile

@@ -0,0 +1,6 @@
+FROM alpine:latest
+COPY data.tar /
+RUN tar xvf /data.tar -C /
+
+FROM alpine:latest
+COPY --from=0 /data/ /data