diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ad487232b46..4f90d4b138c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -231,10 +231,10 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: "3.10" - name: Setup Linux if: runner.os == 'linux' @@ -261,6 +261,21 @@ jobs: pip list shell: bash + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: true + swap-storage: true + + - name: Clean out /tmp directory + run: | + sudo rm -rf /tmp/* + - name: Integration Tests run: | RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=7200 pytest -v --timeout 300 --durations 100 -m "not slow and not combinatorial and not horovod and not llm and $MARKERS" --junitxml pytest.xml tests/integration_tests diff --git a/ludwig/utils/carton_utils.py b/ludwig/utils/carton_utils.py index f19ad84dae2..d03d0a4cd92 100644 --- a/ludwig/utils/carton_utils.py +++ b/ludwig/utils/carton_utils.py @@ -3,7 +3,9 @@ import logging import os import shutil +import sys import tempfile +import traceback from typing import Any, Dict, List import torch @@ -108,26 +110,40 @@ def export_carton(model: LudwigModel, carton_path: str, carton_model_name="ludwi with tempfile.TemporaryDirectory() as tmpdir: # Save the model to a temp dir - input_model_path = os.path.join(tmpdir, "model.pt") + input_model_path: str = os.path.join(tmpdir, "model.pt") torch.jit.save(model_ts, input_model_path) # carton.pack is an async function so we run it and wait until it's complete # See https://pyo3.rs/v0.20.0/ecosystem/async-await#a-note-about-asynciorun for why we wrap it # in another function - async def pack(): - return await carton.pack( - input_model_path, - runner_name="torchscript", - # Any 2.x.x version is okay - # TODO: improve this - required_framework_version="=2", - model_name=carton_model_name, - inputs=_get_input_spec(model), - outputs=_get_output_spec(model), - ) - - loop = asyncio.get_event_loop() - tmp_out_path = loop.run_until_complete(pack()) - - # Move it to the output path - shutil.move(tmp_out_path, carton_path) + async def pack() -> str: + try: + return await carton.pack( + path=input_model_path, + runner_name="torchscript", + # Any 2.x.x version is okay + # TODO: improve this + required_framework_version="=2", + model_name=carton_model_name, + inputs=_get_input_spec(model), + outputs=_get_output_spec(model), + ) + except Exception as e: + exception_message: str = 'An Exception inside "pack()" occurred.\n' + exception_traceback: str = traceback.format_exc() + exception_message += f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".' + sys.stderr.write(exception_message) + sys.stderr.flush() + raise ValueError(exception_message) from e # Re-raise error for calling function to handle. + + try: + tmp_out_path: str = asyncio.get_event_loop().run_until_complete(pack()) + # Move it to the output path + shutil.move(tmp_out_path, carton_path) + except Exception as e: + exception_message: str = 'An Exception inside "export_carton()" occurred.\n' + exception_traceback: str = traceback.format_exc() + exception_message += f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".' + sys.stderr.write(exception_message) + sys.stderr.flush() + raise SystemExit(exception_message) from e # Make sure error is fatal.