added test for bia agent and updated some ingest test wiring and sett…

…ings using

bia-ingest/bia_ingest/api_client.py

@@ -3,12 +3,13 @@
 from bia_integrator_api.api import PrivateApi
 import bia_integrator_api.models as api_models
 import logging
-from bia_ingest.settings import settings
+from bia_ingest.settings import get_settings
 
 logger = logging.getLogger("__main__." + __name__)
 
 
 def get_bia_api_client():
+    settings = get_settings()
     private_api_client = get_client_private(
         username=settings.bia_api_username,
         password=settings.bia_api_password,
@@ -18,6 +19,7 @@ def get_bia_api_client():
 
 
 def get_local_bia_api_client():
+    settings = get_settings()
     api_config = Configuration(host=settings.local_bia_api_basepath)
     private_api = PrivateApi(ApiClient(configuration=api_config))
     try:

bia-ingest/bia_ingest/biostudies/find_bia_studies.py

@@ -6,7 +6,7 @@
 from typing import Optional
 from bia_integrator_api.util import get_client
 from bia_integrator_api.models import Study
-from bia_ingest.settings import settings
+from bia_ingest.settings import get_settings
 import re
 
 logger = logging.getLogger("__main__." + __name__)
@@ -115,7 +115,7 @@ def get_accno(acc_id):
     )
     acc_id_of_interest = [result.accession for result in studies_of_interest]
     logging.info("Fetching all studies from bia api")
-    api_client = get_client(settings.bia_api_basepath)
+    api_client = get_client(get_settings().bia_api_basepath)
     bia_existing_studies = fetch_studies_from_api(api_client, page_size)
     processed_acc_ids = [str(study.accession_id) for study in bia_existing_studies]
     unprocessed_acc_ids = sorted(list(set(acc_id_of_interest) - set(processed_acc_ids)), key=lambda acc_id : get_accno(acc_id))

bia-ingest/bia_ingest/persistence_strategy.py

@@ -12,7 +12,7 @@
 from bia_integrator_api.exceptions import NotFoundException
 import bia_integrator_api.models as api_models
 from bia_ingest.api_client import get_bia_api_client, get_local_bia_api_client
-from bia_ingest.settings import settings
+from bia_ingest.settings import get_settings
 
 logger = logging.getLogger("__main__." + __name__)
 
@@ -84,8 +84,8 @@ def fetch_by_uuid(
 # Persist using API
 class ApiPersister(PersistenceStrategy):
     def __init__(self, api_client: PrivateApi) -> None:
-        assert (
-            isinstance(api_client, PrivateApi) or isinstance(api_client, PublicApi)
+        assert isinstance(api_client, PrivateApi) or isinstance(
+            api_client, PublicApi
         ), f"ApiPersister cannot be created. Expected valid instance of <class 'PrivateApi'> or <class 'PublicApi'>. Got : {type(api_client)} - are your API credentials valid and/or is the API server online?"
         self.api_client = api_client
 
@@ -145,7 +145,7 @@ def persistence_strategy_factory(persistence_mode: PersistenceMode, **kwargs):
         return ApiPersister(api_client=get_local_bia_api_client())
     elif persistence_mode == PersistenceMode.disk:
         return DiskPersister(
-            output_dir_base=settings.bia_data_dir,
+            output_dir_base=get_settings().bia_data_dir,
             accession_id=kwargs["accession_id"],
         )
     else:

bia-ingest/bia_ingest/settings.py

@@ -38,4 +38,6 @@ class Settings(BaseSettings):
     bia_api_password: str = Field("")
 
 
-settings = Settings()
+
+def get_settings():
+    return Settings()

bia-ingest/test/conftest.py

@@ -16,7 +16,16 @@
     BioStudiesProcessingVersion,
 )
 from bia_integrator_api.util import get_client
-from bia_ingest.settings import settings
+import os
+from dotenv.main import dotenv_values
+
+
+def pytest_configure(config: pytest.Config):
+    env_settings = dotenv_values(str(Path(__file__).parents[1] / ".env_template"))
+    os.environ["bia_api_basepath"] = env_settings["local_bia_api_basepath"]
+    os.environ["bia_api_username"] = env_settings["local_bia_api_username"]
+    os.environ["bia_api_password"] = env_settings["local_bia_api_password"]
+
 
 @pytest.fixture
 def test_submission() -> Submission:
@@ -76,7 +85,7 @@ def _mock_request_get(flist_url: str) -> Dict[str, str]:
 
 
 @pytest.fixture
-def mock_search_result(monkeypatch):
+def mock_search_result():
     """Requests.get mocked to read file from disk"""
 
     mock_result = {
@@ -104,16 +113,15 @@ def mock_search_result(monkeypatch):
     }
     search_result = SearchPage(**mock_result)
 
-    def _mock_search_result(url, headers) -> Dict[str, str]:
+    return search_result
 
-        return_value = Mock()
-        return_value.status_code = 200
-        return_value.content = search_result.model_dump_json()
-        return return_value
 
-    monkeypatch.setattr(requests, "get", _mock_search_result)
+@pytest.fixture()
+def get_bia_api_client():
+    return get_client(os.environ.get("bia_api_basepath"))
 
 
 @pytest.fixture()
-def get_bia_api_client():
-    return get_client(settings.local_bia_api_basepath)
+def tmp_bia_data_dir(tmp_path):
+    os.environ["bia_data_dir"] = str(tmp_path)
+    return tmp_path

bia-ingest/test/data/example_bia_agent_study/expected_output/annotation_method/10259a72-db54-4277-928b-6f291d1b9ca1.json

@@ -0,0 +1,19 @@
+{
+  "title_id": "Ground truth segmentation masks",
+  "uuid": "10259a72-db54-4277-928b-6f291d1b9ca1",
+  "version": 0,
+  "model": {
+    "type_name": "AnnotationMethod",
+    "version": 3
+  },
+  "attribute": [],
+  "protocol_description": "Each image contains exactly 20 masks; this is the ground truth for counting. Ground truth for foreground/background segmentation are available as labeled 16bit grayscale images",
+  "annotation_criteria": null,
+  "annotation_coverage": "All data has been annotated.",
+  "transformation_description": null,
+  "spatial_information": null,
+  "method_type": [
+    "other"
+  ],
+  "annotation_source_indicator": null
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/bio_sample/S-BIAD1492/2dbf5050-5704-484c-94ca-98e8e1d08e11.json

@@ -0,0 +1,23 @@
+{
+  "title_id": "Simulated HL-60 cells",
+  "uuid": "2dbf5050-5704-484c-94ca-98e8e1d08e11",
+  "version": 0,
+  "model": {
+    "type_name": "BioSample",
+    "version": 3
+  },
+  "attribute": [],
+  "organism_classification": [
+    {
+      "attribute": [],
+      "common_name": null,
+      "scientific_name": "simulated data",
+      "ncbi_id": null
+    }
+  ],
+  "biological_entity_description": "simulated human promyelocytic leukemia cells (HL-60) stained with DAPI",
+  "experimental_variable_description": [],
+  "extrinsic_variable_description": [],
+  "intrinsic_variable_description": [],
+  "growth_protocol_uuid": null
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/dataset/S-BIAD1492/2d124551-c51a-4d8a-817e-2804f558e059.json

@@ -0,0 +1,25 @@
+{
+  "title_id": "Ground truth segmentation masks",
+  "uuid": "2d124551-c51a-4d8a-817e-2804f558e059",
+  "version": 0,
+  "model": {
+    "type_name": "Dataset",
+    "version": 1
+  },
+  "attribute": [
+    {
+      "provenance": "bia_ingest",
+      "name": "annotation_method_uuid",
+      "value": {
+        "annotation_method_uuid": [
+          "10259a72-db54-4277-928b-6f291d1b9ca1"
+        ]
+      }
+    }
+  ],
+  "description": null,
+  "analysis_method": [],
+  "correlation_method": [],
+  "example_image_uri": [],
+  "submitted_in_study_uuid": "869ff676-bf68-4e2d-869e-7e644cbbec42"
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/dataset/S-BIAD1492/44500cd8-1ad9-4340-ba55-3427a2782486.json

@@ -0,0 +1,58 @@
+{
+  "title_id": "Simulated fluorescence images",
+  "uuid": "44500cd8-1ad9-4340-ba55-3427a2782486",
+  "version": 0,
+  "model": {
+    "type_name": "Dataset",
+    "version": 1
+  },
+  "attribute": [
+    {
+      "provenance": "bia_ingest",
+      "name": "associations",
+      "value": {
+        "associations": [
+          {
+            "image_analysis": null,
+            "image_correlation": null,
+            "biosample": "Simulated HL-60 cells",
+            "image_acquisition": "Simulated fluorescence microscopy",
+            "specimen": "Digital Phantom Generation"
+          }
+        ]
+      }
+    },
+    {
+      "provenance": "bia_ingest",
+      "name": "image_acquisition_protocol_uuid",
+      "value": {
+        "image_acquisition_protocol_uuid": [
+          "5f4095ba-8bd3-4efa-9c7e-18bc35ddf416"
+        ]
+      }
+    },
+    {
+      "provenance": "bia_ingest",
+      "name": "specimen_imaging_preparation_protocol_uuid",
+      "value": {
+        "specimen_imaging_preparation_protocol_uuid": [
+          "c27b92d9-1340-455f-9278-c8663326214e"
+        ]
+      }
+    },
+    {
+      "provenance": "bia_ingest",
+      "name": "bio_sample_uuid",
+      "value": {
+        "bio_sample_uuid": [
+          "2dbf5050-5704-484c-94ca-98e8e1d08e11"
+        ]
+      }
+    }
+  ],
+  "description": "Four subsets (each in high and low signal-to-noise ratio variant) of 30 images each are provided. Each image contains 20 HL-60 cell nuclei, but the nuclei cluster with different probabilities (0%, 25%, 50%, and 75%) in the four subsets.",
+  "analysis_method": [],
+  "correlation_method": [],
+  "example_image_uri": [],
+  "submitted_in_study_uuid": "869ff676-bf68-4e2d-869e-7e644cbbec42"
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/file_reference/S-BIAD1492/0d4d7ee6-d288-4ed2-a4d6-d1dbcde23d80.json

@@ -0,0 +1,25 @@
+{
+  "uuid": "0d4d7ee6-d288-4ed2-a4d6-d1dbcde23d80",
+  "version": 0,
+  "model": {
+    "type_name": "FileReference",
+    "version": 2
+  },
+  "attribute": [
+    {
+      "provenance": "bia_ingest",
+      "name": "attributes_from_biostudies.File",
+      "value": {
+        "attributes": {
+          "Annotation type": "Segmentation masks",
+          "Source image": "BBBC024/BBBC024_v1_c50_lowSNR_images_TIFF/image-final_0012.tif"
+        }
+      }
+    }
+  ],
+  "file_path": "BBBC024/BBBC024_v1_c50_lowSNR_images_TIFF/image-labels_0012.tif",
+  "format": "file",
+  "size_in_bytes": 3195094,
+  "uri": "https://www.ebi.ac.uk/biostudies/files/S-BIAD1492/BBBC024/BBBC024_v1_c50_lowSNR_images_TIFF/image-labels_0012.tif",
+  "submission_dataset_uuid": "2d124551-c51a-4d8a-817e-2804f558e059"
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/file_reference/S-BIAD1492/119b42c3-8ee0-486a-8b3d-9dfff40d09a2.json

@@ -0,0 +1,25 @@
+{
+  "uuid": "119b42c3-8ee0-486a-8b3d-9dfff40d09a2",
+  "version": 0,
+  "model": {
+    "type_name": "FileReference",
+    "version": 2
+  },
+  "attribute": [
+    {
+      "provenance": "bia_ingest",
+      "name": "attributes_from_biostudies.File",
+      "value": {
+        "attributes": {
+          "clustering probability": "0%",
+          "SNR": "high"
+        }
+      }
+    }
+  ],
+  "file_path": "BBBC024/BBBC024_v1_c00_highSNR_images_TIFF/image-final_0012.tif",
+  "format": "file",
+  "size_in_bytes": 61795818,
+  "uri": "https://www.ebi.ac.uk/biostudies/files/S-BIAD1492/BBBC024/BBBC024_v1_c00_highSNR_images_TIFF/image-final_0012.tif",
+  "submission_dataset_uuid": "44500cd8-1ad9-4340-ba55-3427a2782486"
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/file_reference/S-BIAD1492/22550bb0-037b-49b6-946f-7c72f6aa2594.json

@@ -0,0 +1,25 @@
+{
+  "uuid": "22550bb0-037b-49b6-946f-7c72f6aa2594",
+  "version": 0,
+  "model": {
+    "type_name": "FileReference",
+    "version": 2
+  },
+  "attribute": [
+    {
+      "provenance": "bia_ingest",
+      "name": "attributes_from_biostudies.File",
+      "value": {
+        "attributes": {
+          "clustering probability": "75%",
+          "SNR": "low"
+        }
+      }
+    }
+  ],
+  "file_path": "BBBC024/BBBC024_v1_c75_lowSNR_images_TIFF/image-final_0012.tif",
+  "format": "file",
+  "size_in_bytes": 50872008,
+  "uri": "https://www.ebi.ac.uk/biostudies/files/S-BIAD1492/BBBC024/BBBC024_v1_c75_lowSNR_images_TIFF/image-final_0012.tif",
+  "submission_dataset_uuid": "44500cd8-1ad9-4340-ba55-3427a2782486"
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/file_reference/S-BIAD1492/22e3a91e-c02c-4b49-8b9c-7c0a9e657b60.json

@@ -0,0 +1,25 @@
+{
+  "uuid": "22e3a91e-c02c-4b49-8b9c-7c0a9e657b60",
+  "version": 0,
+  "model": {
+    "type_name": "FileReference",
+    "version": 2
+  },
+  "attribute": [
+    {
+      "provenance": "bia_ingest",
+      "name": "attributes_from_biostudies.File",
+      "value": {
+        "attributes": {
+          "Annotation type": "Segmentation masks",
+          "Source image": "BBBC024/BBBC024_v1_c75_lowSNR_images_TIFF/image-final_0012.tif"
+        }
+      }
+    }
+  ],
+  "file_path": "BBBC024/BBBC024_v1_c75_lowSNR_images_TIFF/image-labels_0012.tif",
+  "format": "file",
+  "size_in_bytes": 3217536,
+  "uri": "https://www.ebi.ac.uk/biostudies/files/S-BIAD1492/BBBC024/BBBC024_v1_c75_lowSNR_images_TIFF/image-labels_0012.tif",
+  "submission_dataset_uuid": "2d124551-c51a-4d8a-817e-2804f558e059"
+}

bia-ingest/test/data/example_bia_agent_study/expected_output/file_reference/S-BIAD1492/2b181c80-841d-47c0-a57a-5b0aa60e658c.json

@@ -0,0 +1,25 @@
+{
+  "uuid": "2b181c80-841d-47c0-a57a-5b0aa60e658c",
+  "version": 0,
+  "model": {
+    "type_name": "FileReference",
+    "version": 2
+  },
+  "attribute": [
+    {
+      "provenance": "bia_ingest",
+      "name": "attributes_from_biostudies.File",
+      "value": {
+        "attributes": {
+          "Annotation type": "Segmentation masks",
+          "Source image": "BBBC024/BBBC024_v1_c25_highSNR_images_TIFF/image-final_0012.tif"
+        }
+      }
+    }
+  ],
+  "file_path": "BBBC024/BBBC024_v1_c25_highSNR_images_TIFF/image-labels_0012.tif",
+  "format": "file",
+  "size_in_bytes": 3207776,
+  "uri": "https://www.ebi.ac.uk/biostudies/files/S-BIAD1492/BBBC024/BBBC024_v1_c25_highSNR_images_TIFF/image-labels_0012.tif",
+  "submission_dataset_uuid": "2d124551-c51a-4d8a-817e-2804f558e059"
+}