NER context retrieval: remove context/main sentence separators

CompNet · Sep 5, 2024 · 7fb25c7 · 7fb25c7
1 parent 71ad3a4
commit 7fb25c7
Showing 1 changed file with 0 additions and 4 deletions.
diff --git a/renard/pipeline/ner/retrieval.py b/renard/pipeline/ner/retrieval.py
@@ -55,16 +55,12 @@ def __call__(self, dataset: NERDataset) -> NERDataset:
                 key=lambda m: m.element_i,
             )
             lctx = list(flatten([m.element for m in lctx]))
-            if len(lctx) > 0:
-                lctx += [dataset.tokenizer.sep_token]
 
             rctx = sorted(
                 (m for m in matchs if m.side == "right"),
                 key=lambda m: m.element_i,
             )
             rctx = list(flatten([m.element for m in rctx]))
-            if len(rctx) > 0:
-                rctx = [dataset.tokenizer.sep_token] + rctx
 
             elements_with_context.append((lctx, elt, rctx))