From 94151b98ebfb408a136c70e1e593ab9e3fee3843 Mon Sep 17 00:00:00 2001
From: William Allen <16820599+williamjallen@users.noreply.github.com>
Date: Fri, 3 May 2024 15:17:31 -0400
Subject: [PATCH] Cache submissions while migrations are running

---
 .github/workflows/submit.sh                   |  8 +++++++
 app/Http/Controllers/SubmissionController.php |  5 +++-
 .../Middleware/CheckForMaintenanceMode.php    |  3 ++-
 app/Utils/UnparsedSubmissionProcessor.php     |  4 ++--
 docker/docker-entrypoint.sh                   | 24 ++++++++++++-------
 5 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/submit.sh b/.github/workflows/submit.sh
index e5a862d349..13af5e74fe 100644
--- a/.github/workflows/submit.sh
+++ b/.github/workflows/submit.sh
@@ -19,6 +19,14 @@ echo "database=$database"
 echo "ctest_driver=$ctest_driver"
 echo "submit_type=$submit_type"
 
+# Wait for migrations to finish running by checking for maintenance mode to be lifted
+docker exec cdash bash -c " \
+until [ ! -f /cdash/storage/framework/down ] \
+do \
+  sleep 1 \
+done \
+"
+
 # Suppress any uncommitted changes left after the image build
 docker exec cdash bash -c "cd /cdash && /usr/bin/git checkout ."
 
diff --git a/app/Http/Controllers/SubmissionController.php b/app/Http/Controllers/SubmissionController.php
index 778aaed4ec..8471a2c0a6 100644
--- a/app/Http/Controllers/SubmissionController.php
+++ b/app/Http/Controllers/SubmissionController.php
@@ -100,7 +100,10 @@ private function submitProcess(): Response
         // Check if we can connect to the database before proceeding any further.
         try {
             DB::connection()->getPdo();
-        } catch (\Exception $e) {
+            if (app()->isDownForMaintenance()) {
+                throw new Exception();
+            }
+        } catch (\Exception) {
             // Write a marker file so we know to process these files when the DB comes back up.
             if (!Storage::exists("DB_WAS_DOWN")) {
                 Storage::put("DB_WAS_DOWN", "");
diff --git a/app/Http/Middleware/CheckForMaintenanceMode.php b/app/Http/Middleware/CheckForMaintenanceMode.php
index 35b9824bae..7ae23d7ab0 100755
--- a/app/Http/Middleware/CheckForMaintenanceMode.php
+++ b/app/Http/Middleware/CheckForMaintenanceMode.php
@@ -12,6 +12,7 @@ class CheckForMaintenanceMode extends Middleware
      * @var array
      */
     protected $except = [
-        //
+        '/submit.php',
+        '/ping',
     ];
 }
diff --git a/app/Utils/UnparsedSubmissionProcessor.php b/app/Utils/UnparsedSubmissionProcessor.php
index 83116e7815..155ffcb77c 100644
--- a/app/Utils/UnparsedSubmissionProcessor.php
+++ b/app/Utils/UnparsedSubmissionProcessor.php
@@ -83,7 +83,7 @@ public function postSubmit(): JsonResponse
         // Thus function will throw an exception if invalid data provided
         $this->parseBuildMetadata();
 
-        if ($this->checkDatabaseConnection()) {
+        if ($this->checkDatabaseConnection() && !app()->isDownForMaintenance()) {
             return $this->initializeBuild();
         }
 
@@ -215,7 +215,7 @@ public function putSubmitFile(): JsonResponse
         $this->parseDataFileParameters();
         $ext = pathinfo($this->backupfilename, PATHINFO_EXTENSION);
 
-        $db_up = $this->checkDatabaseConnection();
+        $db_up = $this->checkDatabaseConnection() && !app()->isDownForMaintenance();
         if ($db_up) {
             if (!is_numeric($this->buildid) || $this->buildid < 1) {
                 abort(Response::HTTP_NOT_FOUND, 'Build not found');
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index f6b4773530..0007ede767 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -2,26 +2,34 @@
 
 set -e
 
+# Set job control so we can bg/fg processes
+set -m
+
 php artisan key:check || exit 1
 
 # If the "start-website" argument was provided, start the web server
 if [ "$1" = "start-website" ] ; then
-  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
-    bash /cdash/install.sh --dev --initial-docker-install
-  else
-    bash /cdash/install.sh --initial-docker-install
-  fi
-
-  echo "Starting Apache..."
+   echo "Starting Apache..."
 
   # Start Apache under the current user, in case the current user isn't www-data.  Kubernetes-based systems
-  # typically run under a random user.
+  # typically run under a random user.  We start Apache before running the install scripts so the system can
+  # begin collecting submissions while database migrations run.  Apache starts in the background so the
+  # container gets killed if the migrations fail.
   if [ "$BASE_IMAGE" = "debian" ] ; then
     APACHE_RUN_USER=$(id -u -n) /usr/sbin/apache2ctl -D FOREGROUND
   elif [ "$BASE_IMAGE" = "ubi" ]; then
     /usr/libexec/s2i/run
+  fi & # & puts Apache in the background
+
+  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
+    bash /cdash/install.sh --dev --initial-docker-install
+  else
+    bash /cdash/install.sh --initial-docker-install
   fi
 
+  # Bring Apache to the foreground so the container fails if Apache fails after this point.
+  fg
+
 # If the start-worker argument was provided, start a worker process instead
 elif [ "$1" = "start-worker" ] ; then
   php artisan queue:work