Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lazy load stopwords module to prevent downloading when it's not needed #194

Merged
merged 2 commits into from
Oct 20, 2020
Merged

Lazy load stopwords module to prevent downloading when it's not needed #194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b29159b
Lazy load stopwords module to prevent downloading when it's not needed
hugoabonizio Oct 16, 2020
159eede
Download spaCy English model by default since it's required by many f…
hugoabonizio Oct 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion texthero/nlp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,18 @@

import spacy
import pandas as pd
import en_core_web_sm
from nltk.stem import PorterStemmer, SnowballStemmer
from texthero._types import TextSeries, InputSeries

try:
# If not present, download 'en_core_web_sm'
import en_core_web_sm
except ModuleNotFoundError:
from spacy.cli.download import download as spacy_download

spacy_download("en_core_web_sm")
import en_core_web_sm


@InputSeries(TextSeries)
def named_entities(s: TextSeries, package="spacy") -> pd.Series:
Expand Down
3 changes: 2 additions & 1 deletion texthero/preprocessing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import pandas as pd
import unidecode

from texthero import stopwords as _stopwords
from texthero._types import TokenSeries, TextSeries, InputSeries

from typing import List, Callable, Union
Expand Down Expand Up @@ -329,6 +328,8 @@ def replace_stopwords(
"""

if stopwords is None:
from texthero import stopwords as _stopwords

stopwords = _stopwords.DEFAULT
return s.apply(_replace_stopwords, args=(stopwords, symbol))

Expand Down
9 changes: 0 additions & 9 deletions texthero/stopwords.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,6 @@
nltk.download("stopwords")

from nltk.corpus import stopwords as nltk_en_stopwords

try:
# If not present, download 'en_core_web_sm'
spacy_model = spacy.load("en_core_web_sm")
except OSError:
from spacy.cli.download import download as spacy_download

spacy_download("en_core_web_sm")

from spacy.lang.en import stop_words as spacy_en_stopwords

DEFAULT = set(nltk_en_stopwords.words("english"))
Expand Down