fix: add punkt_tab to nltk LookupError check (#1319)

aimclub · Aug 12, 2024 · 39e960c · 39e960c
1 parent 3d49382
commit 39e960c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/...ore/operations/evaluation/operation_implementations/data_operations/text_preprocessing.py b/...ore/operations/evaluation/operation_implementations/data_operations/text_preprocessing.py
@@ -65,7 +65,7 @@ def transform(self, input_data: InputData) -> OutputData:
 
     @staticmethod
     def _download_nltk_resources():
-        for resource in ['punkt']:
+        for resource in ['punkt', 'punkt_tab']:
             try:
                 nltk.data.find(f'tokenizers/{resource}')
             except LookupError: