i718 Add auxiliary worker to run different queues

.env

@@ -1,3 +1,4 @@
+AUXILIARY_QUEUE_PRIORITY=100
 CHROME_HOSTNAME=chrome
 DATABASE_ADAPTER=postgresql
 DATABASE_HOST=db

app/jobs/application_job.rb

@@ -1,8 +1,18 @@
 # frozen_string_literal: true
 
 class ApplicationJob < ActiveJob::Base
+  before_enqueue :set_auxiliary_queue_priority
+
   # limit to 5 attempts
   retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception|
     # Log error, do nothing, etc.
   end
+
+  private
+
+    def set_auxiliary_queue_priority
+      return unless queue_name.to_sym == :auxiliary
+
+      self.priority = ENV.fetch('AUXILIARY_QUEUE_PRIORITY', 100).to_i
+    end
 end

bin/worker

@@ -9,4 +9,10 @@ else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-exec "echo $DATABASE_URL && bundle exec good_job start"
+if ENV['AUX_WORKER']
+  # Run all queues (including :auxiliary)
+  exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='*' bundle exec good_job"
+else
+  # Run all queues (excluding :auxiliary)
+  exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='-auxiliary' bundle exec good_job"
+end

docker-compose.yml

@@ -23,6 +23,36 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  build:
+    context: .
+    target: hyku-worker
+    args:
+      - SETTINGS__BULKRAX__ENABLED=true
+    cache_from:
+      - ghcr.io/scientist-softserv/adventist-dl/base:${TAG:-latest}
+      - ghcr.io/scientist-softserv/adventist-dl:${TAG:-latest}
+      - ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
+  image: ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
+  depends_on:
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    fits:
+      condition: service_started
+    zoo:
+      condition: service_started
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+
 volumes:
   assets:
   cache:
@@ -180,13 +210,15 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_aux:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     expose:
       - 3000
 
   worker:
-    <<: *app
+    <<: *app-worker
     ################################################################################
     ## Note on commands: by default the commands don't run bundle.  That is to
     ## reduce boot times.  However, when the application is in active
@@ -203,33 +235,11 @@ services:
     ## in your code, bash into the worker container, and interact with the
     ## breakpoints.
     # command: sh -l -c "bundle && tail -f /dev/null"
-    build:
-      context: .
-      target: hyku-worker
-      args:
-        - SETTINGS__BULKRAX__ENABLED=true
-      cache_from:
-        - ghcr.io/scientist-softserv/adventist-dl/base:${TAG:-latest}
-        - ghcr.io/scientist-softserv/adventist-dl:${TAG:-latest}
-        - ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
-    image: ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
-    depends_on:
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      fits:
-        condition: service_started
-      zoo:
-        condition: service_started
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
+
+  worker_aux:
+    <<: *app-worker
+    environment:
+      - AUX_WORKER="true"
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows

ops/dev-deploy.tmpl.yaml

@@ -46,11 +46,10 @@ ingress:
     - host: "*.s2.adventistdigitallibrary.org"
       paths:
         - path: /
-  annotations: {
-    kubernetes.io/ingress.class: "nginx",
-    nginx.ingress.kubernetes.io/proxy-body-size: "0",
-    cert-manager.io/cluster-issuer: wildcard-issuer
-  }
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    cert-manager.io/cluster-issuer: "wildcard-issuer"
   tls:
     - hosts:
         - '*.s2.adventistdigitallibrary.org'
@@ -94,6 +93,8 @@ extraEnvVars: &envVars
     value: /rest
   - name: FEDORA_URL
     value: http://fcrepo.default.svc.cluster.local:8080/rest
+  - name: AUXILIARY_QUEUE_PRIORITY
+    value: "100"
   - name: IN_DOCKER
     value: "true"
   - name: LD_LIBRARY_PATH
@@ -226,7 +227,158 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
+  extraEnvVars: *envVars # GOOD_JOB_QUEUES gets set in bin/worker
+
+# When adding/removing key-value pairs to this block, ensure the
+# corresponding changes are made in the `extraDeploy` block below.
+workerAux:
+  extraEnvVars:
+    # GOOD_JOB_QUEUES gets set in bin/worker
+    - name: AUX_WORKER
+      value: "true"
+
+extraDeploy:
+  - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "hyrax.fullname" . }}-worker-aux
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      spec:
+        replicas: {{ .Values.worker.replicaCount }}
+        selector:
+          matchLabels:
+            {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
+        template:
+          metadata:
+          {{- with .Values.podAnnotations }}
+            annotations:
+              {{- toYaml . | nindent 8 }}
+          {{- end }}
+            labels:
+              {{- include "hyrax.workerSelectorLabels" . | nindent 8 }}
+          spec:
+            initContainers:
+              - name: db-wait
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                env:
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }}
+                command:
+                  - sh
+                  - -c
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
+              {{- if .Values.worker.extraInitContainers }}
+              {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
+              {{- end }}
+            {{- with .Values.imagePullSecrets }}
+            imagePullSecrets:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            serviceAccountName: {{ include "hyrax.serviceAccountName" . }}
+            securityContext:
+              {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+            containers:
+              - name: {{ .Chart.Name }}-worker
+                securityContext:
+                  {{- toYaml .Values.worker.securityContext | nindent 12 }}
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                  {{- if .Values.solrExistingSecret }}
+                  - secretRef:
+                      name: {{ .Values.solrExistingSecret }}
+                  {{- end }}
+                  {{- with .Values.worker.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                env:
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }}
+                {{- if .Values.worker.readinessProbe.enabled }}
+                readinessProbe:
+                  exec:
+                    command:
+                      {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }}
+                  failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
+                  initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
+                  periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
+                  successThreshold: {{ .Values.worker.readinessProbe.successThreshold }}
+                  timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
+                {{- end }}
+                volumeMounts:
+                  - name: derivatives
+                    mountPath: /app/samvera/derivatives
+                  - name: uploads
+                    subPath: file_cache
+                    mountPath: /app/samvera/file_cache
+                  - name: uploads
+                    subPath: uploads
+                    mountPath: /app/samvera/uploads
+                  {{- if .Values.applicationExistingClaim }}
+                  - name: application
+                    mountPath: /app/samvera/hyrax-webapp
+                  {{- end }}
+                  {{- with .Values.worker.extraVolumeMounts }}
+                  {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                resources:
+                  {{- toYaml .Values.worker.resources | nindent 12 }}
+                {{- with .Values.extraContainerConfiguration }}
+                {{- toYaml . | nindent 10 }}
+                {{- end }}
+            volumes:
+              - name: "derivatives"
+                {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.derivativesVolume.existingClaim }}
+                {{- else if .Values.derivativesVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-derivatives
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              - name: "uploads"
+                {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.uploadsVolume.existingClaim }}
+                {{- else if .Values.uploadsVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-uploads
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              {{- if .Values.applicationExistingClaim }}
+              - name: "application"
+                persistentVolumeClaim:
+                  claimName: {{ .Values.applicationExistingClaim }}
+              {{- end }}
+              {{- with .Values.worker.extraVolumes }}
+              {{- toYaml . | nindent 8 }}
+              {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.tolerations }}
+            tolerations:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
 
 podSecurityContext:
   runAsUser: 1001