[BUGFIX] Fixing integration test failures. (#3959)

.github/workflows/pytest.yml

@@ -231,10 +231,10 @@ jobs:
     timeout-minutes: 90
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.9
+      - name: Set up Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: "3.10"
 
       - name: Setup Linux
         if: runner.os == 'linux'
@@ -261,6 +261,21 @@ jobs:
           pip list
         shell: bash
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: true
+          swap-storage: true
+
+      - name: Clean out /tmp directory
+        run: |
+          sudo rm -rf /tmp/*
+
       - name: Integration Tests
         run: |
           RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=7200 pytest -v --timeout 300 --durations 100 -m "not slow and not combinatorial and not horovod and not llm and $MARKERS" --junitxml pytest.xml tests/integration_tests

ludwig/utils/carton_utils.py

@@ -3,7 +3,9 @@
 import logging
 import os
 import shutil
+import sys
 import tempfile
+import traceback
 from typing import Any, Dict, List
 
 import torch
@@ -108,26 +110,40 @@ def export_carton(model: LudwigModel, carton_path: str, carton_model_name="ludwi
 
     with tempfile.TemporaryDirectory() as tmpdir:
         # Save the model to a temp dir
-        input_model_path = os.path.join(tmpdir, "model.pt")
+        input_model_path: str = os.path.join(tmpdir, "model.pt")
         torch.jit.save(model_ts, input_model_path)
 
         # carton.pack is an async function so we run it and wait until it's complete
         # See https://pyo3.rs/v0.20.0/ecosystem/async-await#a-note-about-asynciorun for why we wrap it
         # in another function
-        async def pack():
-            return await carton.pack(
-                input_model_path,
-                runner_name="torchscript",
-                # Any 2.x.x version is okay
-                # TODO: improve this
-                required_framework_version="=2",
-                model_name=carton_model_name,
-                inputs=_get_input_spec(model),
-                outputs=_get_output_spec(model),
-            )
-
-        loop = asyncio.get_event_loop()
-        tmp_out_path = loop.run_until_complete(pack())
-
-        # Move it to the output path
-        shutil.move(tmp_out_path, carton_path)
+        async def pack() -> str:
+            try:
+                return await carton.pack(
+                    path=input_model_path,
+                    runner_name="torchscript",
+                    # Any 2.x.x version is okay
+                    # TODO: improve this
+                    required_framework_version="=2",
+                    model_name=carton_model_name,
+                    inputs=_get_input_spec(model),
+                    outputs=_get_output_spec(model),
+                )
+            except Exception as e:
+                exception_message: str = 'An Exception inside "pack()" occurred.\n'
+                exception_traceback: str = traceback.format_exc()
+                exception_message += f'{type(e).__name__}: "{str(e)}".  Traceback: "{exception_traceback}".'
+                sys.stderr.write(exception_message)
+                sys.stderr.flush()
+                raise ValueError(exception_message) from e  # Re-raise error for calling function to handle.
+
+        try:
+            tmp_out_path: str = asyncio.get_event_loop().run_until_complete(pack())
+            # Move it to the output path
+            shutil.move(tmp_out_path, carton_path)
+        except Exception as e:
+            exception_message: str = 'An Exception inside "export_carton()" occurred.\n'
+            exception_traceback: str = traceback.format_exc()
+            exception_message += f'{type(e).__name__}: "{str(e)}".  Traceback: "{exception_traceback}".'
+            sys.stderr.write(exception_message)
+            sys.stderr.flush()
+            raise SystemExit(exception_message) from e  # Make sure error is fatal.