From c9a63b05dbcda484d0acc517dc76d85e7909b086 Mon Sep 17 00:00:00 2001
From: Liam Keegan <liam@keegan.ch>
Date: Wed, 18 Dec 2024 10:50:27 +0100
Subject: [PATCH] Runner uploads last line of script output as error message if
 script fails

- runner
  - capture and display stdout while script is running
  - upload last line of output as the error_message if script return code is non-zero
- backend
  - add `error_message` to Sample model
  - store the uploaded error message from the runner here (was already stored in the job, but not the sample)
- frontend
  - display `error_message` in Samples table
- resolves #54
---
 backend/src/predicTCR_server/app.py       |  1 +
 backend/src/predicTCR_server/model.py     |  4 ++++
 backend/tests/helpers/flask_test_utils.py |  1 +
 frontend/src/components/SamplesTable.vue  |  2 ++
 frontend/src/utils/types.ts               |  1 +
 runner/src/predicTCR_runner/runner.py     | 20 +++++++++++++++++---
 6 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/backend/src/predicTCR_server/app.py b/backend/src/predicTCR_server/app.py
index b7d079d..8a79e4b 100644
--- a/backend/src/predicTCR_server/app.py
+++ b/backend/src/predicTCR_server/app.py
@@ -285,6 +285,7 @@ def admin_resubmit_sample(sample_id: int):
         sample.trusted_user_result_file_path().unlink(missing_ok=True)
         sample.admin_result_file_path().unlink(missing_ok=True)
         sample.has_results_zip = False
+        sample.error_message = ""
         sample.status = Status.QUEUED
         db.session.commit()
         return jsonify(message="Sample added to the queue")
diff --git a/backend/src/predicTCR_server/model.py b/backend/src/predicTCR_server/model.py
index 046a298..0b9bc4c 100644
--- a/backend/src/predicTCR_server/model.py
+++ b/backend/src/predicTCR_server/model.py
@@ -85,6 +85,7 @@ class Sample(db.Model):
     timestamp_job_end: Mapped[int] = mapped_column(Integer, nullable=False)
     status: Mapped[Status] = mapped_column(Enum(Status), nullable=False)
     has_results_zip: Mapped[bool] = mapped_column(Boolean, nullable=False)
+    error_message: Mapped[str] = mapped_column(String, nullable=False)
 
     def base_path(self) -> pathlib.Path:
         data_path = flask.current_app.config["PREDICTCR_DATA_PATH"]
@@ -216,9 +217,11 @@ def process_result(
     job.timestamp_end = timestamp_now()
     if success:
         job.status = Status.COMPLETED
+        sample.error_message = ""
     else:
         job.status = Status.FAILED
         job.error_message = error_message
+        sample.error_message = error_message
     db.session.commit()
     if sample.has_results_zip:
         logger.warning(f" --> Sample {sample_id} already has results")
@@ -503,6 +506,7 @@ def add_new_sample(
         timestamp_job_end=0,
         status=Status.QUEUED,
         has_results_zip=False,
+        error_message="",
     )
     db.session.add(new_sample)
     db.session.commit()
diff --git a/backend/tests/helpers/flask_test_utils.py b/backend/tests/helpers/flask_test_utils.py
index 3a2da58..0518f93 100644
--- a/backend/tests/helpers/flask_test_utils.py
+++ b/backend/tests/helpers/flask_test_utils.py
@@ -58,6 +58,7 @@ def add_test_samples(app, data_path: pathlib.Path):
                 timestamp_job_end=0,
                 status=status,
                 has_results_zip=False,
+                error_message="",
             )
             db.session.add(new_sample)
             db.session.commit()
diff --git a/frontend/src/components/SamplesTable.vue b/frontend/src/components/SamplesTable.vue
index a438e59..e4cea3c 100644
--- a/frontend/src/components/SamplesTable.vue
+++ b/frontend/src/components/SamplesTable.vue
@@ -105,6 +105,7 @@ function download_samples_as_csv() {
       <fwb-table-head-cell v-if="admin">Runtime</fwb-table-head-cell>
       <fwb-table-head-cell>Inputs</fwb-table-head-cell>
       <fwb-table-head-cell>Results</fwb-table-head-cell>
+      <fwb-table-head-cell>Error message</fwb-table-head-cell>
       <fwb-table-head-cell v-if="admin">Actions</fwb-table-head-cell>
     </fwb-table-head>
     <fwb-table-body>
@@ -155,6 +156,7 @@ function download_samples_as_csv() {
           </template>
           <template v-else> - </template>
         </fwb-table-cell>
+        <fwb-table-cell>{{ sample.error_message }}</fwb-table-cell>
         <fwb-table-cell v-if="admin">
           <fwb-button
             @click="
diff --git a/frontend/src/utils/types.ts b/frontend/src/utils/types.ts
index 9b5fef2..578cd9f 100644
--- a/frontend/src/utils/types.ts
+++ b/frontend/src/utils/types.ts
@@ -9,6 +9,7 @@ export type Sample = {
   timestamp_job_end: number;
   status: string;
   has_results_zip: boolean;
+  error_message: string;
 };
 
 export type User = {
diff --git a/runner/src/predicTCR_runner/runner.py b/runner/src/predicTCR_runner/runner.py
index faf5be4..68aae49 100644
--- a/runner/src/predicTCR_runner/runner.py
+++ b/runner/src/predicTCR_runner/runner.py
@@ -74,6 +74,7 @@ def _upload_result(
         user_results: str,
         trusted_user_results: str,
         admin_results: str,
+        error_message: str,
     ):
         self.logger.info(
             f"...job {self.job_id} {'complete' if success else 'failed'} for sample id {self.sample_id}, uploading {user_results}, {trusted_user_results} and {admin_results}..."
@@ -93,6 +94,7 @@ def _upload_result(
                     "sample_id": self.sample_id,
                     "runner_hostname": self.runner_hostname,
                     "success": success,
+                    "error_message": error_message,
                 },
                 headers=self.auth_header,
                 timeout=30,
@@ -118,8 +120,7 @@ def _run_job(self):
                         f"Failed to download {input_file_type}: {response.content}"
                     )
                     return self._report_job_failed(
-                        f"Failed to download {input_file_type} on {self.runner_hostname}",
-                        "",
+                        f"Failed to download {input_file_type} on {self.runner_hostname}"
                     )
                 input_file_name = f"input.{input_file_type}"
                 self.logger.debug(f"  - writing {input_file_name} to {tmpdir}...")
@@ -141,8 +142,20 @@ def _run_job(self):
                 for result_folder in result_folders:
                     (pathlib.Path(tmpdir) / result_folder).mkdir(exist_ok=True)
                 self.logger.debug(f"  - running {tmpdir}/script.sh...")
-                proc = subprocess.run(args=["./script.sh"], cwd=tmpdir)
+                script_output_last_line = ""
+                with subprocess.Popen(
+                    args=["./script.sh"],
+                    cwd=tmpdir,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    encoding="utf-8",
+                    universal_newlines=True,
+                ) as proc:
+                    for line in proc.stdout:
+                        script_output_last_line = line.strip()
+                        self.logger.debug(f"./script.sh :: {script_output_last_line}")
                 success = proc.returncode == 0
+                error_message = script_output_last_line if not success else ""
                 self.logger.debug(
                     f"     ...{tmpdir}/script.sh {'finished' if success else 'failed'}."
                 )
@@ -154,6 +167,7 @@ def _run_job(self):
                     admin_results=f"{tmpdir}/admin_results.zip",
                     trusted_user_results=f"{tmpdir}/trusted_user_results.zip",
                     user_results=f"{tmpdir}/user_results.zip",
+                    error_message=error_message,
                 )
                 self.poll_interval = 1
             except Exception as e: