Fix export_to_brat when there are spaces before new lines

.github/workflows/tests.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install --upgrade pip
-          pip install -e '.[dev,docs,setup]'
+          pip install -e '.[dev,setup]'
 
       - name: Test with Pytest on Python ${{ matrix.python-version }}
         env:

changelog.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Pending
+
+### Fixed
+- `export_to_brat` issue with spans of entities on multiple lines.
+
 ## v0.8.1 (2023-05-31)
 
 Fix release to allow installation from source

edsnlp/connectors/brat.py

@@ -226,18 +226,17 @@ def export_to_brat(doc, txt_filename, overwrite_txt=False, overwrite_ann=False):
                     ):
                         idx = fragment["begin"]
                         entity_text = doc["text"][fragment["begin"] : fragment["end"]]
-                        for part in entity_text.split("\n"):
-                            begin = idx
-                            end = idx + len(part)
-                            idx = end + 1
-                            if begin != end:
-                                spans.append((begin, end))
+                        # eg: "mon entité  \n  est problématique"
+                        for match in re.finditer(
+                            r"\s*(.+?)(?:( *\n+)+ *|$)", entity_text, flags=re.DOTALL
+                        ):
+                            spans.append((idx + match.start(1), idx + match.end(1)))
                     print(
                         "{}\t{} {}\t{}".format(
                             brat_entity_id,
                             str(entity["label"]),
                             ";".join(" ".join(map(str, span)) for span in spans),
-                            entity_text.replace("\n", " "),
+                            " ".join(doc["text"][begin:end] for begin, end in spans),
                         ),
                         file=f,
                     )

edsnlp/pipelines/ner/umls/patterns.py

@@ -37,10 +37,10 @@ def get_patterns(config: Dict[str, Any]) -> Dict[str, List[str]]:
 
     path, module, filename = get_path(config)
 
-    if path.exists():
+    if path.exists():  # pragma: no cover
         print(f"Loading {filename} from {module.base}")
         return module.load_pickle(name=filename)
-    else:
+    else:  # pragma: no cover
         patterns = download_and_agg_umls(config)
         module.dump_pickle(name=filename, obj=patterns)
         print(f"Saved patterns into {module.base / filename}")
@@ -108,7 +108,7 @@ def download_and_agg_umls(config) -> Dict[str, List[str]]:
     """
 
     api_key = os.getenv("UMLS_API_KEY")
-    if not api_key:
+    if not api_key:  # pragma: no cover
         warnings.warn(
             "You need to define UMLS_API_KEY to download the UMLS. "
             "Get a key by creating an account at "

pyproject.toml

@@ -12,20 +12,21 @@ dependencies = [
     "decorator",
     "loguru",
     "pendulum>=2.1.2",
-    "pydantic>=1.8.2,<2.0.0",
+    "pydantic>=1.10.2,<2.0.0",
     "pysimstring>=1.2.1",
     "regex",
     "rich>=12.0.0",
     "scikit-learn>=1.0.0",
     "spacy>=3.1,<3.5.0",
-    "thinc>=8.0.14,<8.1.11",
+    "thinc>=8.1.10",
     "tqdm",
     "umls-downloader>=0.1.1",
     "numpy>=1.15.0,<1.23.2; python_version<'3.8'",
     "numpy>=1.15.0; python_version>='3.8'",
     "pandas>=1.1.0,<2.0.0; python_version<'3.8'",
     "pandas>=1.4.0,<2.0.0; python_version>='3.8'",
-    "typing_extensions<4.6.0"  # https://github.com/explosion/spaCy/issues/12659
+    "typing_extensions<4.6.0,>=4.0.0; python_version>='3.8'",  # https://github.com/explosion/spaCy/issues/12659
+    "typing_extensions>=4.0.0; python_version<'3.8'"
 ]
 [project.optional-dependencies]
 dev = [
@@ -39,7 +40,7 @@ dev = [
     "pytest>=7.1.0,<8.0.0",
     "pytest-cov>=3.0.0,<4.0.0",
     "pytest-html>=3.1.1,<4.0.0",
-    "torch>=1.0.0,<1.13.0",
+    "torch>=1.0.0",
 ]
 setup = [
     "mlconjug3<3.9.0",
@@ -197,6 +198,22 @@ omit-covered-files = false
 # generate-badge = "."
 # badge-format = "svg"
 
+
+[tool.coverage]
+exclude_lines = [
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "if typing.TYPE_CHECKING:",
+    "@overload",
+    "pragma: no cover",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "def __repr__",
+    "Span.set_extension.*",
+    "Doc.set_extension.*",
+    "Token.set_extension.*",
+]
+
 [tool.cibuildwheel]
 skip = [
     "*p36-*", # Skip Python 3.6

tests/connectors/test_brat.py

@@ -193,7 +193,7 @@ def test_brat(
 A1	etat T1 test
 T2	localisation 39 57	dans le bras droit
 T3	anatomie 47 57	bras droit
-T4	pathologie 75 84;85 98	problème  de locomotion
+T4	pathologie 75 83;85 98	problème de locomotion
 A2	assertion T4 absent
 T5	pathologie 114 117	AVC
 A3	etat T5 passé

tests/pipelines/ner/test_umls.py

@@ -18,6 +18,7 @@
 pattern_config = {"lang": ["FRE"], "sources": ["MSHFRE"]}
 
 
+@pytest.mark.skipif(not os.getenv("UMLS_API_KEY"), reason="No UMLS_API_KEY given")
 def test_get_patterns():
 
     path, _, _ = get_path(pattern_config)
@@ -41,6 +42,7 @@ def test_get_patterns():
     assert len(patterns) == 48587
 
 
+@pytest.mark.skipif(not os.getenv("UMLS_API_KEY"), reason="No UMLS_API_KEY given")
 def test_add_pipe(blank_nlp: Language):
     path, _, _ = get_path(pattern_config)
     if not path.exists():