Merge pull request #2509 from matthewturk/cython_distutils_directives

Use cython distutils directives

.gitignore

@@ -55,10 +55,10 @@ yt/utilities/lib/interpolators.c
 yt/utilities/lib/kdtree.c
 yt/utilities/lib/lenses.c
 yt/utilities/lib/line_integral_convolution.c
-yt/utilities/lib/mesh_construction.cpp
-yt/utilities/lib/mesh_intersection.cpp
-yt/utilities/lib/mesh_samplers.cpp
-yt/utilities/lib/mesh_traversal.cpp
+yt/utilities/lib/embree_mesh/mesh_construction.cpp
+yt/utilities/lib/embree_mesh/mesh_intersection.cpp
+yt/utilities/lib/embree_mesh/mesh_samplers.cpp
+yt/utilities/lib/embree_mesh/mesh_traversal.cpp
 yt/utilities/lib/mesh_triangulation.c
 yt/utilities/lib/mesh_utilities.c
 yt/utilities/lib/pixelization_routines.cpp

setupext.py

@@ -1,17 +1,23 @@
+import os
+import platform
 import contextlib
 import glob
-import os
 import shutil
 import subprocess
 import sys
+from sys import platform as _platform
 import tempfile
+from concurrent.futures import ThreadPoolExecutor as Pool
 
 from distutils import log
-from distutils.ccompiler import new_compiler
+from distutils.ccompiler import new_compiler, CCompiler
 from distutils.sysconfig import customize_compiler
 from distutils.errors import CompileError, LinkError
+from distutils.version import LooseVersion
 from pkg_resources import resource_filename
 from subprocess import Popen, PIPE
+from setuptools.command.build_ext import build_ext as _build_ext
+from setuptools.command.sdist import sdist as _sdist
 
 
 CCODE = """
@@ -124,12 +130,32 @@ def check_for_openmp():
     return using_openmp
 
 
-def check_for_pyembree():
+def check_for_pyembree(std_libs):
+    embree_libs = []
+    embree_aliases = {}
     try:
         fn = resource_filename("pyembree", "rtcore.pxd")
     except ImportError:
-        return None
-    return os.path.dirname(fn)
+        return embree_libs, embree_aliases
+
+    embree_prefix = os.path.abspath(read_embree_location())
+    embree_inc_dir = os.path.join(embree_prefix, 'include')
+    embree_lib_dir = os.path.join(embree_prefix, 'lib')
+    if in_conda_env():
+        conda_basedir = os.path.dirname(os.path.dirname(sys.executable))
+        embree_inc_dir.append(os.path.join(conda_basedir, 'include'))
+        embree_lib_dir.append(os.path.join(conda_basedir, 'lib'))
+
+    if _platform == "darwin":
+        embree_lib_name = "embree.2"
+    else:
+        embree_lib_name = "embree"
+
+    embree_aliases['EMBREE_INC_DIR'] = ['yt/utilities/lib/', embree_inc_dir]
+    embree_aliases['EMBREE_LIB_DIR'] = [embree_lib_dir]
+    embree_aliases['EMBREE_LIBS'] = std_libs + [embree_lib_name]
+    embree_libs += ['yt/utilities/lib/embree_mesh/*.pyx']
+    return embree_libs, embree_aliases
 
 def in_conda_env():
     return any(s in sys.version for s in ("Anaconda", "Continuum"))
@@ -206,3 +232,114 @@ def read_embree_location():
         shutil.rmtree(tmpdir)
 
     return rd
+
+def get_cpu_count():
+    if platform.system() == "Windows":
+        return 0
+
+    cpu_count = os.cpu_count()
+    try:
+        user_max_cores = int(os.getenv('MAX_BUILD_CORES', cpu_count))
+    except ValueError as e:
+        raise ValueError(
+            "MAX_BUILD_CORES must be set to an integer. " +
+            "See above for original error.").with_traceback(e.__traceback__)
+    max_cores = min(cpu_count, user_max_cores)
+    return max_cores
+
+
+def install_ccompiler():
+    def _compile(
+        self, sources, output_dir=None, macros=None, include_dirs=None,
+        debug=0, extra_preargs=None, extra_postargs=None, depends=None,
+    ):
+        """Function to monkey-patch distutils.ccompiler.CCompiler"""
+        macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
+            output_dir, macros, include_dirs, sources, depends, extra_postargs
+        )
+        cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+        for obj in objects:
+            try:
+                src, ext = build[obj]
+            except KeyError:
+                continue
+            self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+        # Return *all* object filenames, not just the ones we just built.
+        return objects
+
+    CCompiler.compile = _compile
+
+def create_build_ext(lib_exts, cythonize_aliases):
+    class build_ext(_build_ext):
+        # subclass setuptools extension builder to avoid importing cython and numpy
+        # at top level in setup.py. See http://stackoverflow.com/a/21621689/1382869
+        def finalize_options(self):
+            try:
+                import cython
+                import numpy
+            except ImportError:
+                raise ImportError(
+    """Could not import cython or numpy. Building yt from source requires
+    cython and numpy to be installed. Please install these packages using
+    the appropriate package manager for your python environment.""")
+            if LooseVersion(cython.__version__) < LooseVersion('0.26.1'):
+                raise RuntimeError(
+    """Building yt from source requires Cython 0.26.1 or newer but
+    Cython %s is installed. Please update Cython using the appropriate
+    package manager for your python environment.""" %
+                    cython.__version__)
+            if LooseVersion(numpy.__version__) < LooseVersion('1.13.3'):
+                raise RuntimeError(
+    """Building yt from source requires NumPy 1.13.3 or newer but
+    NumPy %s is installed. Please update NumPy using the appropriate
+    package manager for your python environment.""" %
+                    numpy.__version__)
+            from Cython.Build import cythonize
+            # Override the list of extension modules
+            self.distribution.ext_modules[:] = cythonize(
+                lib_exts,
+                aliases = cythonize_aliases,
+                compiler_directives={'language_level': 2},
+                nthreads=get_cpu_count(),
+            )
+            _build_ext.finalize_options(self)
+            # Prevent numpy from thinking it is still in its setup process
+            # see http://stackoverflow.com/a/21621493/1382869
+            if isinstance(__builtins__, dict):
+                # sometimes this is a dict so we need to check for that
+                # https://docs.python.org/3/library/builtins.html
+                __builtins__["__NUMPY_SETUP__"] = False
+            else:
+                __builtins__.__NUMPY_SETUP__ = False
+            import numpy
+            self.include_dirs.append(numpy.get_include())
+
+        def build_extensions(self):
+            self.check_extensions_list(self.extensions)
+
+            ncpus = get_cpu_count()
+            if ncpus > 0:
+                with Pool(ncpus) as pool:
+                    pool.map(self.build_extension, self.extensions)
+            else:
+                super().build_extensions()
+
+
+    class sdist(_sdist):
+        # subclass setuptools source distribution builder to ensure cython
+        # generated C files are included in source distribution.
+        # See http://stackoverflow.com/a/18418524/1382869
+        def run(self):
+            # Make sure the compiled Cython files in the distribution are up-to-date
+            from Cython.Build import cythonize
+            cythonize(
+                lib_exts,
+                aliases = cythonize_aliases,
+                compiler_directives={'language_level': 2},
+                nthreads=get_cpu_count(),
+            )
+            _sdist.run(self)
+
+    return build_ext, sdist

tests/test_install_script.py

@@ -36,7 +36,7 @@
 ]
 
 DEPENDENCY_IMPORT_TESTS = {
-    'embree': "from yt.utilities.lib import mesh_traversal",
+    'embree': "from yt.utilities.lib.embree_mesh import mesh_traversal",
 }
 
 

yt/frontends/artio/_artio_caller.pyx

@@ -1,3 +1,5 @@
+# distutils: sources = ARTIO_SOURCE
+# distutils: include_dirs = LIB_DIR_GEOM_ARTIO
 cimport cython
 import numpy as np
 cimport numpy as np

yt/frontends/ramses/io_utils.pyx

@@ -1,3 +1,5 @@
+# distutils: libraries = STD_LIBS
+# distutils: include_dirs = LIB_DIR
 cimport cython
 cimport numpy as np
 import numpy as np

yt/geometry/fake_octree.pyx

@@ -1,3 +1,5 @@
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Make a fake octree, deposit particle at every leaf
 

yt/geometry/grid_container.pyx

@@ -1,3 +1,5 @@
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Matching points on the grid to specific grids
 

yt/geometry/grid_visitors.pyx

@@ -1,3 +1,5 @@
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Grid visitor functions
 

yt/geometry/oct_container.pyx

@@ -1,3 +1,6 @@
+# distutils: sources = yt/utilities/lib/tsearch.c
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Oct container
 

yt/geometry/oct_visitors.pyx

@@ -1,3 +1,5 @@
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Oct visitor functions
 

yt/geometry/particle_deposit.pyx

@@ -1,3 +1,5 @@
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Particle Deposition onto Cells
 

yt/geometry/particle_oct_container.pyx

@@ -1,3 +1,6 @@
+# distutils: include_dirs = LIB_DIR_EWAH
+# distutils: language = c++
+# distutils: libraries = STD_LIBS
 """
 Oct container tuned for Particles
 

yt/geometry/particle_smooth.pyx

@@ -1,3 +1,5 @@
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Particle smoothing in cells
 

yt/geometry/selection_routines.pyx

@@ -1,3 +1,5 @@
+# distutils: include_dirs = LIB_DIR
+# distutils: libraries = STD_LIBS
 """
 Geometry selection routines.
 

yt/utilities/cython_fortran_utils.pyx

@@ -1,3 +1,4 @@
+# distutils: libraries = STD_LIBS
 cimport numpy as np
 import numpy as np
 import cython

yt/utilities/lib/allocation_container.pyx

@@ -1,3 +1,5 @@
+
+# distutils: libraries = STD_LIBS
 """
 An allocation container and memory pool
 

yt/utilities/lib/alt_ray_tracers.pyx

@@ -1,3 +1,4 @@
+# distutils: libraries = STD_LIBS
 """
 
 

yt/utilities/lib/amr_kdtools.pyx

@@ -1,3 +1,5 @@
+
+# distutils: libraries = STD_LIBS
 """
 AMR kD-Tree Cython Tools
 

yt/utilities/lib/basic_octree.pyx

@@ -1,3 +1,5 @@
+
+# distutils: libraries = STD_LIBS
 """
 A refine-by-two AMR-specific octree
 

yt/utilities/lib/bitarray.pyx

@@ -1,3 +1,4 @@
+# distutils: libraries = STD_LIBS
 """
 Bit array functions
 

yt/utilities/lib/bounding_volume_hierarchy.pyx

@@ -1,3 +1,7 @@
+# distutils: libraries = STD_LIBS
+# distutils: include_dirs = LIB_DIR
+# distutils: extra_compile_args = OMP_ARGS
+# distutils: extra_link_args = OMP_ARGS
 cimport cython
 import numpy as np
 cimport numpy as np

yt/utilities/lib/contour_finding.pyx

@@ -1,3 +1,5 @@
+# distutils: libraries = STD_LIBS
+# distutils: include_dirs = LIB_DIR_GEOM
 """
 A two-pass contour finding algorithm
 

yt/utilities/lib/cykdtree/kdtree.pyx

@@ -1,3 +1,9 @@
+# distutils: libraries = STD_LIBS
+# Note that we used to include the empty c_kdtree.cpp file, but that seems to break cythonize.
+# distutils: sources = yt/utilities/lib/cykdtree/c_utils.cpp
+# distutils: depends = yt/utilities/lib/cykdtree/c_kdtree.hpp, yt/utilities/lib/cykdtree/c_utils.hpp
+# distutils: language = c++
+# distutils: extra_compile_args = -std=c++03
 import cython
 import numpy as np
 cimport numpy as np

yt/utilities/lib/cykdtree/utils.pyx

@@ -1,3 +1,8 @@
+# distutils: libraries = STD_LIBS
+# distutils: sources = yt/utilities/lib/cykdtree/c_utils.cpp
+# distutils: depends = yt/utilities/lib/cykdtree/c_utils.hpp
+# distutils: language = c++
+# distutils: extra_compile_args = -std=c++03
 import numpy as np
 cimport numpy as np
 cimport cython

yt/utilities/lib/cyoctree.pyx

@@ -1,3 +1,8 @@
+# distutils: libraries = STD_LIBS
+# distutils: extra_link_args = OMP_ARGS
+# distutils: extra_compile_args = OMP_ARGS
+# distutils: include_dirs = LIB_DIR
+# distutils: language = c++
 """
 CyOctree building, loading and refining routines
 

yt/utilities/lib/depth_first_octree.pyx

@@ -1,3 +1,5 @@
+
+# distutils: libraries = STD_LIBS
 """
 This is a recursive function to return a depth-first octree
 

yt/utilities/lib/distance_queue.pyx

@@ -1,3 +1,5 @@
+
+# distutils: libraries = STD_LIBS
 """
 Distance queue implementation
 

yt/utilities/lib/element_mappings.pyx

@@ -1,3 +1,4 @@
+# distutils: libraries = STD_LIBS
 """
 This file contains coordinate mappings between physical coordinates and those
 defined on unit elements, as well as doing the corresponding intracell

yt/utilities/lib/mesh_construction.pyx → yt/utilities/lib/embree_mesh/mesh_construction.pyx

@@ -1,3 +1,7 @@
+# distutils: include_dirs = EMBREE_INC_DIR
+# distutils: library_dirs = EMBREE_LIB_DIR
+# distutils: libraries = EMBREE_LIBS
+# distutils: language = c++
 """
 This file contains the ElementMesh classes, which represent the target that the
 rays will be cast at when rendering finite element data. This class handles

yt/utilities/lib/mesh_intersection.pxd → yt/utilities/lib/embree_mesh/mesh_intersection.pxd

@@ -1,7 +1,7 @@
 cimport pyembree.rtcore as rtc
 cimport pyembree.rtcore_ray as rtcr
 cimport pyembree.rtcore_geometry as rtcg
-from yt.utilities.lib.mesh_construction cimport Patch, Tet_Patch
+from .mesh_construction cimport Patch, Tet_Patch
 cimport cython
 
 cdef void patchIntersectFunc(Patch* patches,