Capture errors properly in the http API

cdsobs/api_rest/endpoints.py

@@ -21,6 +21,7 @@
 from cdsobs.service_definition.api import get_service_definition
 from cdsobs.service_definition.service_definition_models import ServiceDefinition
 from cdsobs.storage import S3Client
+from cdsobs.utils.exceptions import DataNotFoundException, SizeError
 from cdsobs.utils.utils import get_database_session
 
 router = APIRouter()
@@ -55,12 +56,26 @@ def get_object_urls_and_check_size(
 ) -> list[str]:
     # Query the storage to get the URLS of the files that contain the data requested
     retrieve_args = retrieve_payload.retrieve_args
-    catalogue_repository = CatalogueRepository(session.catalogue_session)
-    entries = _get_catalogue_entries(catalogue_repository, retrieve_args)
+    try:
+        catalogue_repository = CatalogueRepository(session.catalogue_session)
+        entries = _get_catalogue_entries(catalogue_repository, retrieve_args)
+    except DataNotFoundException as e:
+        raise HTTPException(status_code=500, detail=f"Error: {e}")
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Error: Observations API failed: {e}"
+        )
     s3client = S3Client.from_config(session.cdsobs_config.s3config)
-    object_urls = get_urls_and_check_size(
-        entries, retrieve_args, retrieve_payload.config.size_limit, s3client.base
-    )
+    try:
+        object_urls = get_urls_and_check_size(
+            entries, retrieve_args, retrieve_payload.config.size_limit, s3client.base
+        )
+    except SizeError as e:
+        raise HTTPException(status_code=500, detail=f"Error: {e}")
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Error: Observations API failed: {e}"
+        )
     return object_urls
 
 

cdsobs/cli/_catalogue_explorer.py

@@ -16,8 +16,9 @@
 from cdsobs.observation_catalogue.repositories.cads_dataset import CadsDatasetRepository
 from cdsobs.observation_catalogue.repositories.catalogue import CatalogueRepository
 from cdsobs.observation_catalogue.schemas.catalogue import CliCatalogueFilters
-from cdsobs.utils.logutils import ConfigError, get_logger
+from cdsobs.utils.logutils import get_logger
 
+from ..utils.exceptions import ConfigError
 from ._utils import (
     PAGE_SIZE,
     CliException,

cdsobs/cli/_copy_dataset.py

@@ -17,7 +17,8 @@
 from cdsobs.observation_catalogue.repositories.catalogue import CatalogueRepository
 from cdsobs.observation_catalogue.schemas.catalogue import CatalogueSchema
 from cdsobs.storage import S3Client
-from cdsobs.utils.logutils import ConfigError, get_logger
+from cdsobs.utils.exceptions import ConfigError
+from cdsobs.utils.logutils import get_logger
 
 logger = get_logger(__name__)
 

cdsobs/cli/_delete_dataset.py

@@ -20,7 +20,7 @@
     CliCatalogueFilters,
 )
 from cdsobs.storage import S3Client
-from cdsobs.utils.logutils import ConfigError
+from cdsobs.utils.exceptions import ConfigError
 
 console = Console()
 

cdsobs/config.py

@@ -7,7 +7,7 @@
 import yaml
 from pydantic_core.core_schema import ValidationInfo
 
-from cdsobs.utils.logutils import ConfigError
+from cdsobs.utils.exceptions import ConfigError
 from cdsobs.utils.types import LatTileSize, LonTileSize, TimeTileSize
 
 

cdsobs/observation_catalogue/repositories/catalogue.py

@@ -5,7 +5,7 @@
 
 from cdsobs.observation_catalogue.models import Catalogue
 from cdsobs.observation_catalogue.repositories.base_repository import BaseRepository
-from cdsobs.utils.logutils import CatalogueException
+from cdsobs.utils.exceptions import CatalogueException
 
 
 class CatalogueRepository(BaseRepository):

cdsobs/retrieve/retrieve_services.py

@@ -12,7 +12,8 @@
 from cdsobs.observation_catalogue.schemas.constraints import ConstraintsSchema
 from cdsobs.retrieve.filter_datasets import between
 from cdsobs.retrieve.models import RetrieveArgs
-from cdsobs.utils.logutils import SizeError, get_logger
+from cdsobs.utils.exceptions import DataNotFoundException, SizeError
+from cdsobs.utils.logutils import get_logger, sizeof_fmt
 
 logger = get_logger(__name__)
 
@@ -177,7 +178,9 @@ def _check_data_size(
     estimated_data_size = estimate_data_size(entries, retrieve_args)
     logger.info(f"Estimated size of the data to retrieve is {estimated_data_size}B")
     if estimated_data_size > size_limit:
-        raise SizeError("Requested data size is over the size limit.")
+        raise SizeError(
+            f"Requested data size is over the size limit ({sizeof_fmt(size_limit)})."
+        )
 
 
 def _get_catalogue_entries(
@@ -189,7 +192,7 @@ def _get_catalogue_entries(
         sort=True,
     )
     if len(entries) == 0:
-        raise RuntimeError(
+        raise DataNotFoundException(
             "No entries found in catalogue for this parameter combination."
         )
     logger.info("Retrieved list of required partitions from the catalogue.")

cdsobs/utils/exceptions.py

@@ -0,0 +1,14 @@
+class SizeError(Exception):
+    pass
+
+
+class ConfigError(Exception):
+    pass
+
+
+class CatalogueException(Exception):
+    pass
+
+
+class DataNotFoundException(RuntimeError):
+    pass

cdsobs/utils/logutils.py

@@ -14,16 +14,12 @@
 LogLevel = Literal["NOTSET", "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"]
 
 
-class SizeError(Exception):
-    pass
-
-
-class ConfigError(Exception):
-    pass
-
-
-class CatalogueException(Exception):
-    pass
+def sizeof_fmt(num, suffix="B"):
+    for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"):
+        if abs(num) < 1024.0:
+            return f"{num:3.1f} {unit}{suffix}"
+        num /= 1024.0
+    return f"{num:.1f} Yi{suffix}"
 
 
 def configure_logger() -> None: