feat!: pydantic v2 fhir.resources v7 update, dropping Medication crea…

…tion (#193)

* feat!: pydantic v2 fhir.resources v7 update, dropping Medication creation (#184)

* chore(deps): updated to pydantic v2 and fhir.resources v7

* chore(deps): updated images

* feat!: no longer create Medication resources due to incompatibility with MII profiles

* chore: format, isort and fix endpoint

* ci: ready for beta releases

* fix

* trailing whitespace

* typing

* lints

* pyright

* fix: round to 5 decimal places by default (#185)

* fix: round to 5 decimal places by default

* fix flake8 line too long

* fix: no longer fill MedicationStatement.dosage (#186)

.github/workflows/ci.yaml

@@ -2,11 +2,15 @@ name: ci
 
 on:
   push:
-    branches: [master]
+    branches:
+      - master
+      - beta
   release:
     types: [created]
   pull_request:
-    branches: [master]
+    branches:
+      - master
+      - beta
 
 permissions: read-all
 

.github/workflows/validate-fhir-resources.yaml

@@ -2,40 +2,32 @@ name: validate-fhir-resources
 
 on:
   pull_request:
-    branches: [master]
+    branches:
+      - master
+      - beta
   push:
-    branches: [master]
+    branches:
+      - master
+      - beta
 
 permissions: read-all
 
 jobs:
   validate-fhir-resource:
     name: Validate FHIR resources
     runs-on: ubuntu-22.04
+    container: ghcr.io/miracum/ig-build-tools:v2.1.5@sha256:4571ddd801664e2ee8883ae9c22f88d2c5dfe1175b1e93f042ae8bfa9a7e185a
     steps:
       - name: Checkout code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - name: Setup .NET SDK
-        uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0
-        with:
-          dotnet-version: 6.0.x
-
-      - name: Setup Java JDK
-        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
-        with:
-          distribution: "microsoft"
-          java-version: "17"
-
-      - name: Install Firely.Terminal
-        run: |
-          dotnet tool install --global Firely.Terminal --version 3.1.0
-          fhir --help
-
       - name: Restore FHIR package dependencies
         run: |
           fhir restore
 
       - name: Validate generated FHIR resources
         run: |
-          for fhir_file in tests/__snapshots__/**/*.fhir.json; do fhir validate --verbose --fail "$fhir_file"; done
+          set -e
+          for fhir_file in tests/__snapshots__/**/*.fhir.json; do
+            fhir validate --verbose --fail "$fhir_file";
+          done

README.md

@@ -178,17 +178,13 @@ pip install git+https://github.com/miracum/ahd2fhir@master
 
 ```python
 import json
-from fhir.resources.documentreference import DocumentReference
-from fhir.resources.reference import Reference
-from ahd2fhir.mappers import ahd_to_medication, ahd_to_condition
+from fhir.resources.R4B.documentreference import DocumentReference
+from fhir.resources.R4B.reference import Reference
+from ahd2fhir.mappers import ahd_to_condition
 
 with open('tests/resources/ahd/payload_1.json') as json_resource:
     ahd_payload = json.load(json_resource)
 
-# Get medications directly from from payload dictionary
-medications = ahd_to_medication.get_fhir_medication(ahd_payload)
-
-
 # Create Patient reference and DocumentReference
 pat = FHIRReference(**{'reference': f'Patient/f1234'})
 doc = DocumentReference.construct()

ahd2fhir/config.py

@@ -1,7 +1,8 @@
 from os import path
 
 from aiokafka.helpers import create_ssl_context
-from pydantic import BaseSettings, root_validator, validator
+from pydantic import model_validator
+from pydantic_settings import BaseSettings
 
 TLS_ROOT_DIR = "/opt/kafka-certs/"
 
@@ -37,8 +38,8 @@ class Config:
 class KafkaSettings(BaseSettings):
     input_topic: str = "fhir.documents"
     output_topic: str = "fhir.nlp-results"
-    consumer = KafkaConsumerSettings()
-    producer = KafkaProducerSettings()
+    consumer: KafkaConsumerSettings = KafkaConsumerSettings()
+    producer: KafkaProducerSettings = KafkaProducerSettings()
 
     # Kafka-related settings
     bootstrap_servers: str = "localhost:9094"
@@ -57,7 +58,7 @@ class KafkaSettings(BaseSettings):
     # For using SASL without SSL certificates the *file args need to be None.
     # Otherwise AIOKafkaClient will try to parse them even if they
     # consist of an empty string.
-    @validator("ssl_cafile", "ssl_certfile", "ssl_keyfile")
+    @model_validator(mode="before")
     def parse_to_none(cls, v):
         return None if v in ["", "None", 0, False] else v
 
@@ -105,6 +106,11 @@ class FhirSystemSettings(BaseSettings):
         "https://www.medizininformatik-initiative.de/fhir/core/"
         + "modul-medikation/StructureDefinition/medikationsliste"
     )
+    medication_profile: str = (
+        "https://www.medizininformatik-initiative.de/"
+        + "fhir/core/modul-medikation/StructureDefinition/Medication"
+    )
+    ahd_to_fhir_base_url: str = "https://fhir.miracum.org/ahd2fhir"
 
     class Config:
         env_prefix = "fhir_systems_"
@@ -137,13 +143,12 @@ class Settings(BaseSettings):
     # FHIR systems
     fhir_systems: FhirSystemSettings = FhirSystemSettings()
 
-    @root_validator(skip_on_failure=True)
-    @classmethod
-    def check_ahd_auth(cls, values):
-        if values["ahd_api_token"] == "":
-            if values["ahd_username"] == "" or values["ahd_password"] == "":
+    @model_validator(mode="after")
+    def check_ahd_auth(self):
+        if self.ahd_api_token == "":  # nosec CWE-259
+            if self.ahd_username == "" or self.ahd_password == "":  # nosec CWE-259
                 raise ValueError(
                     "If ahd_api_token is unset, both ahd_username "
                     + "and ahd_password need to be specified."
                 )
-        return values
+        return self

ahd2fhir/kafka_setup.py

@@ -3,8 +3,8 @@
 import aiokafka
 import structlog
 from aiokafka.structs import ConsumerRecord
-from fhir.resources.bundle import Bundle
-from fhir.resources.documentreference import DocumentReference
+from fhir.resources.R4B.bundle import Bundle
+from fhir.resources.R4B.documentreference import DocumentReference
 
 from ahd2fhir import config
 from ahd2fhir.utils.resource_handler import ResourceHandler, TransientError

ahd2fhir/main.py

@@ -1,14 +1,14 @@
 import asyncio
 import os
 from functools import lru_cache
-from typing import Union
+from typing import Any, Dict, Union
 
 import structlog
 from averbis import Client, Pipeline
 from fastapi import Depends, FastAPI, status
 from fastapi.encoders import jsonable_encoder
-from fhir.resources.bundle import Bundle
-from fhir.resources.documentreference import DocumentReference
+from fhir.resources.R4B.bundle import Bundle
+from fhir.resources.R4B.documentreference import DocumentReference
 from prometheus_fastapi_instrumentator import Instrumentator
 from starlette.responses import JSONResponse
 
@@ -92,10 +92,24 @@ async def health():
 
 @app.post("/fhir/$analyze-document")
 async def analyze_document(
-    payload: Union[Bundle, DocumentReference],
+    # directly using Union[DocumentReference, Bundle]
+    # fails in the latest fhir.resources/pydantic
+    payload: Dict[Any, Any],
     resource_handler: ResourceHandler = Depends(get_resource_handler),
 ):
-    result = await analyze_resource(payload, resource_handler)
+    resource = None
+    try:
+        if payload["resourceType"] == "Bundle":
+            resource = Bundle.validate(payload)
+        if payload["resourceType"] == "DocumentReference":
+            resource = DocumentReference.validate(payload)
+    except ValueError:
+        return JSONResponse(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            content="The input resources are likely malformed",
+        )
+
+    result = await analyze_resource(resource, resource_handler)
 
     return JSONResponse(
         status_code=status.HTTP_200_OK,

ahd2fhir/mappers/ahd_to_condition.py

@@ -1,13 +1,13 @@
 import datetime
 import re
 
-from fhir.resources.codeableconcept import CodeableConcept
-from fhir.resources.coding import Coding
-from fhir.resources.condition import Condition
-from fhir.resources.documentreference import DocumentReference
-from fhir.resources.fhirtypes import DateTime
-from fhir.resources.identifier import Identifier
-from fhir.resources.meta import Meta
+from fhir.resources.R4B.codeableconcept import CodeableConcept
+from fhir.resources.R4B.coding import Coding
+from fhir.resources.R4B.condition import Condition
+from fhir.resources.R4B.documentreference import DocumentReference
+from fhir.resources.R4B.fhirtypes import DateTime
+from fhir.resources.R4B.identifier import Identifier
+from fhir.resources.R4B.meta import Meta
 from structlog import get_logger
 
 from ahd2fhir import config
@@ -28,8 +28,6 @@
     "BOTH": ("51440002", "Right and left"),
 }
 
-FHIR_SYSTEMS = config.FhirSystemSettings()
-
 EXTRACT_YEAR_FROM_ICD_REGEX = r"ICD.*_(?P<version>\d{4})"
 
 

ahd2fhir/mappers/ahd_to_list.py

@@ -1,19 +1,20 @@
 from hashlib import sha256
 
-from fhir.resources.codeableconcept import CodeableConcept
-from fhir.resources.coding import Coding
-from fhir.resources.documentreference import DocumentReference
-from fhir.resources.fhirprimitiveextension import FHIRPrimitiveExtension
-from fhir.resources.identifier import Identifier
-from fhir.resources.list import List
-from fhir.resources.meta import Meta
-from fhir.resources.reference import Reference
+from fhir.resources.R4B.codeableconcept import CodeableConcept
+from fhir.resources.R4B.coding import Coding
+from fhir.resources.R4B.documentreference import DocumentReference
+from fhir.resources.R4B.fhirprimitiveextension import FHIRPrimitiveExtension
+from fhir.resources.R4B.identifier import Identifier
+from fhir.resources.R4B.list import List
+from fhir.resources.R4B.meta import Meta
+from fhir.resources.R4B.reference import Reference
 from structlog import get_logger
 
 from ahd2fhir import config
 from ahd2fhir.mappers.ahd_to_medication_statement import (
     get_medication_statement_from_annotation,
 )
+from ahd2fhir.utils.const import AHD_TYPE_MEDICATION
 
 log = get_logger()
 
@@ -46,7 +47,11 @@
 def get_medication_statement_reference(annotation, document_reference):
     medication_statement = get_medication_statement_from_annotation(
         annotation, document_reference
-    )[0]["statement"]
+    )
+
+    if medication_statement is None:
+        return None
+
     medication_reference = Reference.construct()
     medication_reference.type = f"{medication_statement.resource_type}"
     medication_reference.identifier = medication_statement.identifier[0]
@@ -96,7 +101,7 @@ def get_medication_list_from_document_reference(
     med_entries: dict[str, list] = {"ADMISSION": [], "DISCHARGE": [], "INPATIENT": []}
 
     for annotation in annotation_results:
-        if annotation["type"] != "de.averbis.types.health.Medication":
+        if annotation["type"] != AHD_TYPE_MEDICATION:
             continue
 
         status = annotation.get("status")
@@ -109,8 +114,14 @@ def get_medication_list_from_document_reference(
             )
             continue
 
+        medication_statement_reference = get_medication_statement_reference(
+            annotation, document_reference
+        )
+        if medication_statement_reference is None:
+            continue
+
         med_entry = {
-            "item": get_medication_statement_reference(annotation, document_reference),
+            "item": medication_statement_reference,
         }
 
         # lst-3 "An entry date can only be used if the mode of the list is "working""