Add sirenia to Github Actions CI; Significant rework of docker compose

- All 3 test apps use the same container image.
- dassie stays in hyrax-webapp, koppie/sirenia live in hyrax-koppie.
- db_migrate service moved to dev-entrypoint in web service.
- Worker waits to start until rails app is up, should reduce bundle install racing.
- hyrax-engine-dev target now based on hyrax-worker-base.
- Chrome is still a pain and randomly stops responding.

.dassie/.env

@@ -2,7 +2,7 @@ ANALYTICS_START_DATE=2021-08-21
 BUNDLE_GEMFILE=Gemfile.dassie
 BUNDLE_PATH=/app/bundle
 CH12N_TOOL=fits_servlet
-CHROME_HEADLESS_MODE=false
+CHROME_HEADLESS_MODE=true
 DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL=true
 DATABASE_TEST_URL=postgresql://hyrax_user:hyrax_password@postgres/hyrax_test?pool=5
 DATABASE_URL=postgresql://hyrax_user:hyrax_password@postgres/hyrax?pool=5

.dockerignore

@@ -14,9 +14,11 @@ artifacts/*
 coverage/*
 chart/*
 
-Gemfile.lock
-.dassie/Gemfile.lock
-.koppie/Gemfile.lock
+Gemfile*lock
+*/Gemfile*lock
 
-node_modules/*
+
+node_modules
+*/node_modules
 yarn.lock
+*/yarn.lock

.github/workflows/lint-build-test.yml

@@ -1,5 +1,5 @@
 name: 'Lint Build Test'
-run-name: Link Build Test of ${{ github.ref_name }} by @${{ github.actor }}
+run-name: Lint Build Test of ${{ github.ref_name }} by @${{ github.actor }}
 on:
   push:
     branches:
@@ -44,39 +44,35 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        ci_test_app: [dassie,koppie]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Build and export
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
-          build-args: |
-            APP_PATH=.${{ matrix.ci_test_app }}
           cache-from: type=gha
           cache-to: type=gha, mode=max
           context: .
           target: hyrax-engine-dev
-          tags: samvera/${{ matrix.ci_test_app }}:latest,samvera/${{ matrix.ci_test_app }}:${{ github.sha }}
-          outputs: type=docker,dest=/tmp/${{ matrix.ci_test_app }}-${{ github.sha }}.tar
+          tags: samvera/hyrax-dev:latest,samvera/hyrax-dev:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/hyrax-dev-${{ github.sha }}.tar
       - name: Upload built image artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.ci_test_app }}-image
-          path: /tmp/${{ matrix.ci_test_app }}-${{ github.sha }}.tar
+          name: hyrax-dev
+          path: /tmp/hyrax-dev-${{ github.sha }}.tar
 
   test:
     needs: build
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        ci_test_app: [dassie,koppie]
-        ci_node_total: [8]
-        ci_node_index: [0,1,2,3,4,5,6,7]
+        ci_test_app: [dassie,koppie,sirenia]
+        ci_node_total: [4]
+        ci_node_index: [0,1,2,3]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -89,11 +85,11 @@ jobs:
       - name: Download built image artifact
         uses: actions/download-artifact@v4
         with:
-          name: ${{ matrix.ci_test_app }}-image
+          name: hyrax-dev
           path: /tmp
       - name: Start containers
         run: |
-          docker load --input /tmp/${{ matrix.ci_test_app }}-${{ github.sha }}.tar
+          docker load --input /tmp/hyrax-dev-${{ github.sha }}.tar
           docker image ls -a
           docker compose -f docker-compose-${{ matrix.ci_test_app }}.yml up -d --quiet-pull --pull missing --no-build
       - name: RSpec
@@ -107,7 +103,7 @@ jobs:
         if: always()
         uses: jwalton/gh-docker-logs@v2
         with:
-          images: 'seleniarm/standalone-chromium'
+          images: 'selenium/standalone-chromium,postgres,fcrepo/fcrepo,solr,redis,samvera/fitsservlet,samvera/fcrepo4'
       - name: Move Test Files
         if: always()
         env:

.github/workflows/test-results.yml

@@ -1,5 +1,5 @@
 name: Test Results
-
+run-name: Test Results of ${{ github.ref_name }} by @${{ github.actor }}
 on:
   workflow_run:
     workflows: ["Lint Build Test"]

.koppie/.env

@@ -2,7 +2,7 @@ APP_NAME=koppie
 BUNDLE_GEMFILE=Gemfile.koppie
 BUNDLE_PATH=/app/bundle
 CH12N_TOOL=fits_servlet
-CHROME_HEADLESS_MODE=false
+CHROME_HEADLESS_MODE=true
 DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL=true
 DATABASE_TEST_URL=postgresql://hyrax_user:hyrax_password@postgres/hyrax_test?pool=5
 DATABASE_URL=postgresql://hyrax_user:hyrax_password@postgres/hyrax?pool=5
@@ -14,9 +14,9 @@ DB_USERNAME=hyrax_user
 FCREPO_URL=http://fedoraAdmin:fedoraAdmin@fcrepo:8080/fcrepo/rest
 FITS_SERVLET_URL=http://fits:8080/fits
 HUB_URL=http://chrome:4444/wd/hub
-HYRAX_DERIVATIVES_PATH=/app/samvera/hyrax-webapp/derivatives/
+HYRAX_DERIVATIVES_PATH=/app/samvera/hyrax-koppie/derivatives/
 HYRAX_ENGINE_PATH=/app/samvera/hyrax-engine
-HYRAX_UPLOAD_PATH=/app/samvera/hyrax-webapp/uploads/
+HYRAX_UPLOAD_PATH=/app/samvera/hyrax-koppie/uploads/
 HYRAX_VALKYRIE=true
 IN_DOCKER=true
 KARMA_BROWSER=remote-chromium
@@ -29,7 +29,7 @@ POSTGRES_PASSWORD=hyrax_password
 POSTGRES_USER=hyrax_user
 RACK_ENV=development
 RAILS_ENV=development
-RAILS_ROOT=.koppie
+RAILS_ROOT=/app/samvera/hyrax-koppie
 RAILS_LOG_TO_STDOUT=false
 RAILS_QUEUE=sidekiq
 REDIS_HOST=redis

Dockerfile

@@ -1,5 +1,5 @@
 ARG ALPINE_VERSION=3.19
-ARG RUBY_VERSION=3.2.4
+ARG RUBY_VERSION=3.2.6
 
 FROM ruby:$RUBY_VERSION-alpine$ALPINE_VERSION as hyrax-base
 
@@ -91,40 +91,22 @@ ONBUILD RUN bundle install --jobs "$(nproc)"
 ONBUILD RUN RAILS_ENV=production SECRET_KEY_BASE=`bin/rake secret` DB_ADAPTER=nulldb DATABASE_URL='postgresql://fake' bundle exec rake assets:precompile
 
 
-FROM hyrax-base as hyrax-engine-dev
+FROM hyrax-worker-base as hyrax-engine-dev
 
-USER root
-RUN apk --no-cache add bash \
-  ffmpeg \
-  mediainfo \
-  perl
 USER app
-
-ARG APP_PATH=.dassie
 ARG BUNDLE_WITHOUT=
-
 ENV HYRAX_ENGINE_PATH /app/samvera/hyrax-engine
 
-COPY --chown=1001:101 $APP_PATH /app/samvera/hyrax-webapp
+COPY --chown=1001:101 .dassie /app/samvera/hyrax-webapp
+COPY --chown=1001:101 .koppie /app/samvera/hyrax-koppie
 COPY --chown=1001:101 . /app/samvera/hyrax-engine
 
 RUN bundle -v && \
-  bundle install --jobs "$(nproc)" && \
-  cd $HYRAX_ENGINE_PATH && \
-  bundle install --jobs "$(nproc)" && \
-  yarn && yarn cache clean
-
-RUN RAILS_ENV=production SECRET_KEY_BASE='fakesecret1234' DB_ADAPTER=nulldb DATABASE_URL='postgresql://fake' bundle exec rake assets:precompile
-
-
-FROM hyrax-worker-base as hyrax-engine-dev-worker
-
-ARG APP_PATH=.dassie
-ARG BUNDLE_WITHOUT=
-
-ENV HYRAX_ENGINE_PATH /app/samvera/hyrax-engine
-
-COPY --chown=1001:101 $APP_PATH /app/samvera/hyrax-webapp
-COPY --chown=1001:101 . /app/samvera/hyrax-engine
-
-RUN bundle install --jobs "$(nproc)"
+  BUNDLE_GEMFILE=Gemfile.dassie bundle install --jobs "$(nproc)" && yarn && \
+  cd ../hyrax-koppie && BUNDLE_GEMFILE=Gemfile.koppie bundle install --jobs "$(nproc)" && yarn && \
+  cd $HYRAX_ENGINE_PATH && bundle install --jobs "$(nproc)" && yarn && \
+  yarn cache clean
+
+# RUN RAILS_ENV=production SECRET_KEY_BASE='fakesecret1234' DB_ADAPTER=nulldb DATABASE_URL='postgresql://fake' bundle exec rake assets:precompile
+ENTRYPOINT ["dev-entrypoint.sh"]
+CMD ["bundle", "exec", "puma", "-v", "-b", "tcp://0.0.0.0:3000"]

bin/db-wait.sh

@@ -5,9 +5,9 @@ port=$(printf "%s\n" "$1"| cut -d : -f 2)
 
 shift 1
 
+echo "silently waiting for $host:$port"
 while ! nc -z "$host" "$port"
 do
-  echo "waiting for $host:$port"
   sleep 1
 done
 

bin/dev-entrypoint.sh

@@ -1,13 +1,17 @@
 #!/bin/sh
 set -e
 
-mkdir -p /app/samvera/hyrax-webapp/tmp/pids
-rm -f /app/samvera/hyrax-webapp/tmp/pids/*
+mkdir -p $RAILS_ROOT/tmp/pids
+rm -f $RAILS_ROOT/tmp/pids/*
+
+RUBY_MAJOR=$(ruby -e "puts /^(?'major'\d+)\.(?'minor'\d+)\.(?'patch'\d+)/.match(RUBY_VERSION)[:major]")
 
 # Copy gems installed in the image to the dev bundle
 mkdir -p /app/bundle/ruby/$RUBY_MAJOR.0
 cp -Rn /usr/local/bundle/* /app/bundle/ruby/$RUBY_MAJOR.0
 bundle install
 
+db-migrate-seed.sh
+
 # Run the command
 exec "$@"

bin/worker-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+# Wait for web app to avoid racing during bundle install
+db-wait.sh web:3000
+
+# Run the command
+exec "$@"

docker-compose-dassie.yml

@@ -6,8 +6,7 @@ services:
       args:
         - EXTRA_APK_PACKAGES=git less
         - BUNDLE_GEMFILE=Gemfile.dassie
-    image: samvera/dassie
-    entrypoint: dev-entrypoint.sh
+    image: samvera/hyrax-dev
     command: sh -c 'bundle exec puma -v -b tcp://0.0.0.0:3000'
     stdin_open: true
     tty: true
@@ -18,7 +17,6 @@ services:
       - RAILS_ROOT=/app/samvera/hyrax-webapp
     depends_on:
       - chrome
-      - db_migrate
       - fcrepo
       - fits
       - memcached
@@ -41,14 +39,13 @@ services:
       - hyrax
 
   worker:
-    image: samvera/dassie
-    entrypoint: dev-entrypoint.sh
+    image: samvera/hyrax-dev
+    entrypoint: worker-entrypoint.sh
     command: sh -c 'bundle exec sidekiq'
     user: root
     env_file:
       - .dassie/.env
     depends_on:
-      - db_migrate
       - fcrepo
       - fits
       - memcached
@@ -67,27 +64,8 @@ services:
     networks:
       - hyrax
 
-  db_migrate:
-    image: samvera/dassie
-    user: root
-    env_file:
-      - .dassie/.env
-    entrypoint: dev-entrypoint.sh
-    command: db-migrate-seed.sh
-    depends_on:
-      - postgres
-    volumes:
-      - ./bin:/app/samvera
-      - .dassie:/app/samvera/hyrax-webapp
-      - .:/app/samvera/hyrax-engine
-      - bundle:/app/bundle
-      - rails-public:/app/samvera/hyrax-webapp/public
-      - rails-tmp:/app/samvera/hyrax-webapp/tmp
-    networks:
-      - hyrax
-
   chrome:
-    image: seleniarm/standalone-chromium:4
+    image: selenium/standalone-chromium:4
     environment:
       - START_XVFB=false
       - SE_NODE_SESSION_TIMEOUT=800
@@ -103,7 +81,7 @@ services:
       - hyrax
 
   postgres:
-    image: postgres:15
+    image: postgres:15-alpine
     environment:
       - POSTGRES_USER=hyrax_user
       - POSTGRES_PASSWORD=hyrax_password