Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove native Python implementations of Cython extensions #2630

Merged
merged 23 commits into from
Oct 25, 2019
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
update setup.py
mpenkov committed Oct 19, 2019

Verified

This commit was signed with the committer’s verified signature.
acao Rikki Schulte
commit 06dafeb8ab5d251de1178956a6dd63c896a54bf1
145 changes: 43 additions & 102 deletions setup.py
Original file line number Diff line number Diff line change
@@ -29,61 +29,7 @@


class custom_build_ext(build_ext):
"""Allow C extension building to fail.

The C extension speeds up word2vec and doc2vec training, but is not essential.
"""

warning_message = """
********************************************************************
WARNING: %s could not
be compiled. No C extensions are essential for gensim to run,
although they do result in significant speed improvements for some modules.
%s

Here are some hints for popular operating systems:

If you are seeing this message on Linux you probably need to
install GCC and/or the Python development package for your
version of Python.

Debian and Ubuntu users should issue the following command:

$ sudo apt-get install build-essential python-dev

RedHat, CentOS, and Fedora users should issue the following command:

$ sudo yum install gcc python-devel

If you are seeing this message on OSX please read the documentation
here:

http://api.mongodb.org/python/current/installation.html#osx
********************************************************************
"""

def run(self):
try:
build_ext.run(self)
except Exception:
e = sys.exc_info()[1]
sys.stdout.write('%s\n' % str(e))
warnings.warn(
self.warning_message +
"Extension modules" +
"There was an issue with your platform configuration - see above.")

def build_extension(self, ext):
name = ext.name
try:
build_ext.build_extension(self, ext)
except Exception:
e = sys.exc_info()[1]
sys.stdout.write('%s\n' % str(e))
warnings.warn(
self.warning_message +
"The %s extension module" % (name,) +
"The output above this warning shows how the compilation failed.")
"""Solve chicken-and-egg problem with numpy."""

# the following is needed to be able to add numpy's include dirs... without
# importing numpy directly in this script, before it's actually installed!
@@ -101,9 +47,6 @@ def finalize_options(self):
self.include_dirs.append(numpy.get_include())


model_dir = os.path.join(os.path.dirname(__file__), 'gensim', 'models')
gensim_dir = os.path.join(os.path.dirname(__file__), 'gensim')

cmdclass = {'build_ext': custom_build_ext}

WHEELHOUSE_UPLOADER_COMMANDS = {'fetch_artifacts', 'upload_all'}
@@ -290,28 +233,28 @@ def finalize_options(self):
if sys.version_info == (2, 7):
docs_testenv.insert(0, 'doctools==0.14')

ext_modules = [
Extension('gensim.models.word2vec_inner',
sources=['./gensim/models/word2vec_inner.c'],
include_dirs=[model_dir]),
Extension('gensim.models.doc2vec_inner',
sources=['./gensim/models/doc2vec_inner.c'],
include_dirs=[model_dir]),
Extension('gensim.corpora._mmreader',
sources=['./gensim/corpora/_mmreader.c']),
Extension('gensim.models.fasttext_inner',
sources=['./gensim/models/fasttext_inner.c'],
include_dirs=[model_dir]),
Extension('gensim.models._utils_any2vec',
sources=['./gensim/models/_utils_any2vec.c'],
include_dirs=[model_dir]),
Extension('gensim._matutils',
sources=['./gensim/_matutils.c']),
Extension('gensim.models.nmf_pgd',
sources=['./gensim/models/nmf_pgd.c'])
]

if not (os.name == 'nt' and sys.version_info[0] < 3):
def make_c_extensions():
import numpy
include_dirs = [numpy.get_include()]

extensions = {
'gensim.models.word2vec_inner': './gensim/models/word2vec_inner.pyx',
'gensim.models.doc2vec_inner': './gensim/models/doc2vec_inner.pyx',
'gensim.corpora._mmreader': './gensim/corpora/_mmreader.pyx',
'gensim.models.fasttext_inner': './gensim/models/fasttext_inner.pyx',
'gensim.models._utils_any2vec': './gensim/models/_utils_any2vec.pyx',
'gensim._matutils': './gensim/_matutils.pyx',
'gensim.models.nmf_pgd': './gensim/models/nmf_pgd.pyx',
}
for module, source in extensions.items():
yield Extension(module, sources=[source], include_dirs=include_dirs)


def make_cpp_extensions():
import numpy
include_dirs = [numpy.get_include()]

extra_args = []
system = platform.system()

@@ -320,29 +263,26 @@ def finalize_options(self):
elif system == 'Darwin':
extra_args.extend(['-stdlib=libc++', '-std=c++11'])

ext_modules.append(
Extension('gensim.models.word2vec_corpusfile',
sources=['./gensim/models/word2vec_corpusfile.cpp'],
language='c++',
extra_compile_args=extra_args,
extra_link_args=extra_args)
)

ext_modules.append(
Extension('gensim.models.fasttext_corpusfile',
sources=['./gensim/models/fasttext_corpusfile.cpp'],
language='c++',
extra_compile_args=extra_args,
extra_link_args=extra_args)
)

ext_modules.append(
Extension('gensim.models.doc2vec_corpusfile',
sources=['./gensim/models/doc2vec_corpusfile.cpp'],
language='c++',
extra_compile_args=extra_args,
extra_link_args=extra_args)
)
extensions = {
'gensim.models.word2vec_corpusfile': './gensim/models/word2vec_corpusfile.pyx',
'gensim.models.fasttext_corpusfile': './gensim/models/fasttext_corpusfile.pyx',
'gensim.models.fasttext_corpusfile': './gensim/models/fasttext_corpusfile.pyx',
'gensim.models.doc2vec_corpusfile': './gensim/models/doc2vec_corpusfile.pyx',
}

for module, source in extensions.items():
yield Extension(
module,
sources=[source],
language='c++',
extra_compile_args=extra_args,
extra_link_args=extra_args,
include_dirs=include_dirs,
)


ext_modules = list(make_c_extensions()) + list(make_cpp_extensions())


setup(
name='gensim',
@@ -395,6 +335,7 @@ def finalize_options(self):
'scipy >= 0.18.1',
'six >= 1.5.0',
'smart_open >= 1.8.1',
'cython',
piskvorky marked this conversation as resolved.
Show resolved Hide resolved
],
tests_require=linux_testenv,
extras_require={