From 3583ffbc9a4a8df557e7193fedf8a5e7e2f479f4 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Sun, 29 Mar 2020 14:07:38 -0500 Subject: [PATCH 01/17] Converting yt/geometry to use cythonize --- setup.py | 47 ++++---------------------- yt/geometry/fake_octree.pyx | 2 ++ yt/geometry/grid_container.pyx | 2 ++ yt/geometry/grid_visitors.pyx | 2 ++ yt/geometry/oct_container.pyx | 3 ++ yt/geometry/oct_visitors.pyx | 2 ++ yt/geometry/particle_deposit.pyx | 2 ++ yt/geometry/particle_oct_container.pyx | 3 ++ yt/geometry/particle_smooth.pyx | 2 ++ yt/geometry/selection_routines.pyx | 2 ++ 10 files changed, 27 insertions(+), 40 deletions(-) diff --git a/setup.py b/setup.py index db45e97ca30..f4b3c729d53 100644 --- a/setup.py +++ b/setup.py @@ -83,46 +83,13 @@ def _compile( else: std_libs = ["m"] -cython_extensions = [ - Extension("yt.geometry.grid_visitors", - ["yt/geometry/grid_visitors.pyx"], - include_dirs=["yt/utilities/lib"], - libraries=std_libs), - Extension("yt.geometry.grid_container", - ["yt/geometry/grid_container.pyx"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs), - Extension("yt.geometry.oct_container", - ["yt/geometry/oct_container.pyx", - "yt/utilities/lib/tsearch.c"], - include_dirs=["yt/utilities/lib"], - libraries=std_libs), - Extension("yt.geometry.oct_visitors", - ["yt/geometry/oct_visitors.pyx"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs), - Extension("yt.geometry.particle_oct_container", - ["yt/geometry/particle_oct_container.pyx"], - include_dirs=["yt/utilities/lib/", - "yt/utilities/lib/ewahboolarray"], - language="c++", - libraries=std_libs), - Extension("yt.geometry.selection_routines", - ["yt/geometry/selection_routines.pyx"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs), - Extension("yt.geometry.particle_deposit", - ["yt/geometry/particle_deposit.pyx"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs), - Extension("yt.geometry.particle_smooth", - ["yt/geometry/particle_smooth.pyx"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs), - Extension("yt.geometry.fake_octree", - ["yt/geometry/fake_octree.pyx"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs), +from Cython.Build import cythonize +cythonize_aliases = {'LIB_DIR': 'yt/utilities/lib/', + 'STD_LIBS': std_libs} + +cython_extensions = cythonize("yt/geometry/*.pyx", aliases = cythonize_aliases, + language_level = 2) +cython_extensions += [ Extension("yt.utilities.lib.autogenerated_element_samplers", ["yt/utilities/lib/autogenerated_element_samplers.pyx"], include_dirs=["yt/utilities/lib/"]), diff --git a/yt/geometry/fake_octree.pyx b/yt/geometry/fake_octree.pyx index e69dc92165f..9c972bc1648 100644 --- a/yt/geometry/fake_octree.pyx +++ b/yt/geometry/fake_octree.pyx @@ -1,3 +1,5 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Make a fake octree, deposit particle at every leaf diff --git a/yt/geometry/grid_container.pyx b/yt/geometry/grid_container.pyx index 73f995a911e..32e3e039d10 100644 --- a/yt/geometry/grid_container.pyx +++ b/yt/geometry/grid_container.pyx @@ -1,3 +1,5 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Matching points on the grid to specific grids diff --git a/yt/geometry/grid_visitors.pyx b/yt/geometry/grid_visitors.pyx index 5c77bfacf57..0ddc95e3c72 100644 --- a/yt/geometry/grid_visitors.pyx +++ b/yt/geometry/grid_visitors.pyx @@ -1,3 +1,5 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Grid visitor functions diff --git a/yt/geometry/oct_container.pyx b/yt/geometry/oct_container.pyx index 7fac2de545a..02851e676ac 100644 --- a/yt/geometry/oct_container.pyx +++ b/yt/geometry/oct_container.pyx @@ -1,3 +1,6 @@ +# distutils: sources = yt/utilities/lib/tsearch.c +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Oct container diff --git a/yt/geometry/oct_visitors.pyx b/yt/geometry/oct_visitors.pyx index f8212b6dff8..e27f57bd280 100644 --- a/yt/geometry/oct_visitors.pyx +++ b/yt/geometry/oct_visitors.pyx @@ -1,3 +1,5 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Oct visitor functions diff --git a/yt/geometry/particle_deposit.pyx b/yt/geometry/particle_deposit.pyx index bc9c36f0443..3009f713044 100644 --- a/yt/geometry/particle_deposit.pyx +++ b/yt/geometry/particle_deposit.pyx @@ -1,3 +1,5 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Particle Deposition onto Cells diff --git a/yt/geometry/particle_oct_container.pyx b/yt/geometry/particle_oct_container.pyx index 9aeec58dae5..684bad6bbd8 100644 --- a/yt/geometry/particle_oct_container.pyx +++ b/yt/geometry/particle_oct_container.pyx @@ -1,3 +1,6 @@ +# distutils: include_dirs = LIB_DIR, yt/utilities/lib/ewahboolarray +# distutils: language = c++ +# distutils: libraries = STD_LIBS """ Oct container tuned for Particles diff --git a/yt/geometry/particle_smooth.pyx b/yt/geometry/particle_smooth.pyx index 469aa335a47..af98bb30198 100644 --- a/yt/geometry/particle_smooth.pyx +++ b/yt/geometry/particle_smooth.pyx @@ -1,3 +1,5 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Particle smoothing in cells diff --git a/yt/geometry/selection_routines.pyx b/yt/geometry/selection_routines.pyx index 694ceb9448a..64fce6c65f1 100644 --- a/yt/geometry/selection_routines.pyx +++ b/yt/geometry/selection_routines.pyx @@ -1,3 +1,5 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Geometry selection routines. From 512d5afdc1bea675a7b3c47f3fbdf95ab084fbd7 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Sun, 29 Mar 2020 14:44:02 -0500 Subject: [PATCH 02/17] Fix particle_oct_container build --- setup.py | 1 + yt/geometry/particle_oct_container.pyx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4b3c729d53..bacc0bea124 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ def _compile( from Cython.Build import cythonize cythonize_aliases = {'LIB_DIR': 'yt/utilities/lib/', + 'LIB_DIR_EWAH': ['yt/utilities/lib/', 'yt/utilities/lib/ewahboolarray/'], 'STD_LIBS': std_libs} cython_extensions = cythonize("yt/geometry/*.pyx", aliases = cythonize_aliases, diff --git a/yt/geometry/particle_oct_container.pyx b/yt/geometry/particle_oct_container.pyx index 684bad6bbd8..6bfc6f8a710 100644 --- a/yt/geometry/particle_oct_container.pyx +++ b/yt/geometry/particle_oct_container.pyx @@ -1,4 +1,4 @@ -# distutils: include_dirs = LIB_DIR, yt/utilities/lib/ewahboolarray +# distutils: include_dirs = LIB_DIR_EWAH # distutils: language = c++ # distutils: libraries = STD_LIBS """ From 34c48239b54c97f35485817e850ba0f8cfc8e18d Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 10:48:31 -0500 Subject: [PATCH 03/17] Extending lib_exts cythonize --- setup.py | 12 +++++------- yt/utilities/lib/allocation_container.pyx | 2 ++ yt/utilities/lib/amr_kdtools.pyx | 2 ++ yt/utilities/lib/basic_octree.pyx | 2 ++ yt/utilities/lib/depth_first_octree.pyx | 2 ++ yt/utilities/lib/distance_queue.pyx | 2 ++ yt/utilities/lib/fortran_reader.pyx | 2 ++ yt/utilities/lib/image_utilities.pyx | 2 ++ yt/utilities/lib/interpolators.pyx | 2 ++ yt/utilities/lib/lenses.pyx | 2 ++ yt/utilities/lib/mesh_utilities.pyx | 2 ++ yt/utilities/lib/particle_mesh_operations.pyx | 2 ++ yt/utilities/lib/points_in_volume.pyx | 2 ++ yt/utilities/lib/quad_tree.pyx | 2 ++ 14 files changed, 31 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index bacc0bea124..16a64b73eb1 100644 --- a/setup.py +++ b/setup.py @@ -224,17 +224,15 @@ def _compile( libraries=std_libs), ] -lib_exts = [ +lib_exts = ["yt/utilities/lib/{}.pyx".format(_) for _ in [ "particle_mesh_operations", "depth_first_octree", "fortran_reader", "interpolators", "basic_octree", "image_utilities", "points_in_volume", "quad_tree", "mesh_utilities", "amr_kdtools", "lenses", "distance_queue", "allocation_container", -] -for ext_name in lib_exts: - cython_extensions.append( - Extension("yt.utilities.lib.{}".format(ext_name), - ["yt/utilities/lib/{}.pyx".format(ext_name)], - libraries=std_libs)) +]] + +cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, + language_level = 2) lib_exts = ["write_array", "ragged_arrays", "line_integral_convolution"] for ext_name in lib_exts: diff --git a/yt/utilities/lib/allocation_container.pyx b/yt/utilities/lib/allocation_container.pyx index 4bb61517cc6..9022731e05b 100644 --- a/yt/utilities/lib/allocation_container.pyx +++ b/yt/utilities/lib/allocation_container.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ An allocation container and memory pool diff --git a/yt/utilities/lib/amr_kdtools.pyx b/yt/utilities/lib/amr_kdtools.pyx index 464345cdc87..fbf39501e68 100644 --- a/yt/utilities/lib/amr_kdtools.pyx +++ b/yt/utilities/lib/amr_kdtools.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ AMR kD-Tree Cython Tools diff --git a/yt/utilities/lib/basic_octree.pyx b/yt/utilities/lib/basic_octree.pyx index c95c47c5538..59a56831680 100644 --- a/yt/utilities/lib/basic_octree.pyx +++ b/yt/utilities/lib/basic_octree.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ A refine-by-two AMR-specific octree diff --git a/yt/utilities/lib/depth_first_octree.pyx b/yt/utilities/lib/depth_first_octree.pyx index 45280ae4924..7b7b6de5a44 100644 --- a/yt/utilities/lib/depth_first_octree.pyx +++ b/yt/utilities/lib/depth_first_octree.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ This is a recursive function to return a depth-first octree diff --git a/yt/utilities/lib/distance_queue.pyx b/yt/utilities/lib/distance_queue.pyx index 15deb631252..6eb1aa9730e 100644 --- a/yt/utilities/lib/distance_queue.pyx +++ b/yt/utilities/lib/distance_queue.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Distance queue implementation diff --git a/yt/utilities/lib/fortran_reader.pyx b/yt/utilities/lib/fortran_reader.pyx index a6a77793947..0cb90710a1f 100644 --- a/yt/utilities/lib/fortran_reader.pyx +++ b/yt/utilities/lib/fortran_reader.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Simple readers for fortran unformatted data, specifically for the Tiger code. diff --git a/yt/utilities/lib/image_utilities.pyx b/yt/utilities/lib/image_utilities.pyx index a83e6ea9412..75687df4c1e 100644 --- a/yt/utilities/lib/image_utilities.pyx +++ b/yt/utilities/lib/image_utilities.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Utilities for images """ diff --git a/yt/utilities/lib/interpolators.pyx b/yt/utilities/lib/interpolators.pyx index 9bf6f2c64b7..42c0e445697 100644 --- a/yt/utilities/lib/interpolators.pyx +++ b/yt/utilities/lib/interpolators.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Simple interpolators diff --git a/yt/utilities/lib/lenses.pyx b/yt/utilities/lib/lenses.pyx index 0e36474f2b0..f435143747d 100644 --- a/yt/utilities/lib/lenses.pyx +++ b/yt/utilities/lib/lenses.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Functions for computing the extent of lenses and whatnot diff --git a/yt/utilities/lib/mesh_utilities.pyx b/yt/utilities/lib/mesh_utilities.pyx index 5a0bddfd0dc..4fb3763f800 100644 --- a/yt/utilities/lib/mesh_utilities.pyx +++ b/yt/utilities/lib/mesh_utilities.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Utilities for unstructured and semi-structured meshes diff --git a/yt/utilities/lib/particle_mesh_operations.pyx b/yt/utilities/lib/particle_mesh_operations.pyx index 81c3e6ff99c..129b8a63e97 100644 --- a/yt/utilities/lib/particle_mesh_operations.pyx +++ b/yt/utilities/lib/particle_mesh_operations.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Simple integrators for the radiative transfer equation diff --git a/yt/utilities/lib/points_in_volume.pyx b/yt/utilities/lib/points_in_volume.pyx index 041e1775f29..d4649d27dfc 100644 --- a/yt/utilities/lib/points_in_volume.pyx +++ b/yt/utilities/lib/points_in_volume.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ Checks for points contained in a volume diff --git a/yt/utilities/lib/quad_tree.pyx b/yt/utilities/lib/quad_tree.pyx index 07e99375664..d00b3fdb28d 100644 --- a/yt/utilities/lib/quad_tree.pyx +++ b/yt/utilities/lib/quad_tree.pyx @@ -1,3 +1,5 @@ + +# distutils: libraries = STD_LIBS """ A refine-by-two AMR-specific quadtree From fb4b208e7f2d0245ae70dc3854f069bd1945d7b1 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 11:02:45 -0500 Subject: [PATCH 04/17] Continuing to convert to use cythonize --- setup.py | 21 ++------------------- yt/utilities/lib/bitarray.pyx | 1 + yt/utilities/lib/fnv_hash.pyx | 2 ++ yt/utilities/lib/particle_kdtree_tools.pyx | 1 + 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 16a64b73eb1..0361d6937a4 100644 --- a/setup.py +++ b/setup.py @@ -94,9 +94,6 @@ def _compile( Extension("yt.utilities.lib.autogenerated_element_samplers", ["yt/utilities/lib/autogenerated_element_samplers.pyx"], include_dirs=["yt/utilities/lib/"]), - Extension("yt.utilities.lib.bitarray", - ["yt/utilities/lib/bitarray.pyx"], - libraries=std_libs), Extension("yt.utilities.lib.bounding_volume_hierarchy", ["yt/utilities/lib/bounding_volume_hierarchy.pyx"], include_dirs=["yt/utilities/lib/"], @@ -131,10 +128,6 @@ def _compile( libraries=std_libs, language="c++", extra_compile_arg=["-std=c++03"]), - Extension("yt.utilities.lib.fnv_hash", - ["yt/utilities/lib/fnv_hash.pyx"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs), Extension("yt.utilities.lib.geometry_utils", ["yt/utilities/lib/geometry_utils.pyx"], extra_compile_args=omp_args, @@ -146,12 +139,6 @@ def _compile( include_dirs=["yt/utilities/lib/"], libraries=std_libs, depends=["yt/utilities/lib/fixed_interpolator.h"]), - Extension("yt.utilities.lib.mesh_triangulation", - ["yt/utilities/lib/mesh_triangulation.pyx"], - depends=["yt/utilities/lib/mesh_triangulation.h"]), - Extension("yt.utilities.lib.particle_kdtree_tools", - ["yt/utilities/lib/particle_kdtree_tools.pyx"], - language="c++"), Extension("yt.utilities.lib.bounded_priority_queue", ["yt/utilities/lib/bounded_priority_queue.pyx"]), Extension("yt.utilities.lib.pixelization_routines", @@ -229,17 +216,13 @@ def _compile( "interpolators", "basic_octree", "image_utilities", "points_in_volume", "quad_tree", "mesh_utilities", "amr_kdtools", "lenses", "distance_queue", "allocation_container", + "write_array", "ragged_arrays", "line_integral_convolution", + "bitarray", "fnv_hash", "mesh_triangulation", "particle_kdtree_tools" ]] cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, language_level = 2) -lib_exts = ["write_array", "ragged_arrays", "line_integral_convolution"] -for ext_name in lib_exts: - cython_extensions.append( - Extension("yt.utilities.lib.{}".format(ext_name), - ["yt/utilities/lib/{}.pyx".format(ext_name)])) - extensions = [ Extension("yt.frontends.artio._artio_caller", ["yt/frontends/artio/_artio_caller.pyx"] + diff --git a/yt/utilities/lib/bitarray.pyx b/yt/utilities/lib/bitarray.pyx index c046b51532c..8f7c1f385c4 100644 --- a/yt/utilities/lib/bitarray.pyx +++ b/yt/utilities/lib/bitarray.pyx @@ -1,3 +1,4 @@ +# distutils: libraries = STD_LIBS """ Bit array functions diff --git a/yt/utilities/lib/fnv_hash.pyx b/yt/utilities/lib/fnv_hash.pyx index d3ba7b0fc98..69af4749afd 100644 --- a/yt/utilities/lib/fnv_hash.pyx +++ b/yt/utilities/lib/fnv_hash.pyx @@ -1,3 +1,5 @@ +# distutils: libraries = STD_LIBS +# distutils: include_dirs = LIB_DIR """ Fast hashing routines diff --git a/yt/utilities/lib/particle_kdtree_tools.pyx b/yt/utilities/lib/particle_kdtree_tools.pyx index 96a6fa6a91c..17d41765e68 100644 --- a/yt/utilities/lib/particle_kdtree_tools.pyx +++ b/yt/utilities/lib/particle_kdtree_tools.pyx @@ -1,3 +1,4 @@ +# distutils: language = c++ """ Cython tools for working with the PyKDTree particle KDTree. From feb8d76a9560b2d4a65502bbffa509915f549952 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 11:10:32 -0500 Subject: [PATCH 05/17] Move remaining vanilla libs into cythonize --- setup.py | 25 +++++-------------------- yt/utilities/lib/alt_ray_tracers.pyx | 1 + yt/utilities/lib/element_mappings.pyx | 1 + yt/utilities/lib/primitives.pyx | 1 + 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 0361d6937a4..da56aa8682c 100644 --- a/setup.py +++ b/setup.py @@ -88,12 +88,8 @@ def _compile( 'LIB_DIR_EWAH': ['yt/utilities/lib/', 'yt/utilities/lib/ewahboolarray/'], 'STD_LIBS': std_libs} -cython_extensions = cythonize("yt/geometry/*.pyx", aliases = cythonize_aliases, - language_level = 2) +cython_extensions = [] cython_extensions += [ - Extension("yt.utilities.lib.autogenerated_element_samplers", - ["yt/utilities/lib/autogenerated_element_samplers.pyx"], - include_dirs=["yt/utilities/lib/"]), Extension("yt.utilities.lib.bounding_volume_hierarchy", ["yt/utilities/lib/bounding_volume_hierarchy.pyx"], include_dirs=["yt/utilities/lib/"], @@ -139,8 +135,6 @@ def _compile( include_dirs=["yt/utilities/lib/"], libraries=std_libs, depends=["yt/utilities/lib/fixed_interpolator.h"]), - Extension("yt.utilities.lib.bounded_priority_queue", - ["yt/utilities/lib/bounded_priority_queue.pyx"]), Extension("yt.utilities.lib.pixelization_routines", ["yt/utilities/lib/pixelization_routines.pyx", "yt/utilities/lib/pixelization_constants.c"], @@ -156,11 +150,6 @@ def _compile( extra_link_args=omp_args, libraries=std_libs, language='c++'), - Extension("yt.utilities.lib.primitives", - ["yt/utilities/lib/primitives.pyx"], - libraries=std_libs), - Extension("yt.utilities.lib.cosmology_time", - ["yt/utilities/lib/cosmology_time.pyx"]), Extension("yt.utilities.lib.origami", ["yt/utilities/lib/origami.pyx", "yt/utilities/lib/origami_tags.c"], @@ -191,12 +180,6 @@ def _compile( include_dirs=["yt/utilities/lib/"], libraries=std_libs, depends=["yt/utilities/lib/fixed_interpolator.h"]), - Extension("yt.utilities.lib.element_mappings", - ["yt/utilities/lib/element_mappings.pyx"], - libraries=std_libs), - Extension("yt.utilities.lib.alt_ray_tracers", - ["yt/utilities/lib/alt_ray_tracers.pyx"], - libraries=std_libs), Extension("yt.utilities.lib.misc_utilities", ["yt/utilities/lib/misc_utilities.pyx"], extra_compile_args=omp_args, @@ -211,13 +194,15 @@ def _compile( libraries=std_libs), ] -lib_exts = ["yt/utilities/lib/{}.pyx".format(_) for _ in [ +lib_exts = ["yt/geometry/*.pyx"] + ["yt/utilities/lib/{}.pyx".format(_) for _ in [ "particle_mesh_operations", "depth_first_octree", "fortran_reader", "interpolators", "basic_octree", "image_utilities", "points_in_volume", "quad_tree", "mesh_utilities", "amr_kdtools", "lenses", "distance_queue", "allocation_container", "write_array", "ragged_arrays", "line_integral_convolution", - "bitarray", "fnv_hash", "mesh_triangulation", "particle_kdtree_tools" + "bitarray", "fnv_hash", "mesh_triangulation", "particle_kdtree_tools", + "primitives", "cosmology_time", "element_mappings", "alt_ray_tracers", + "bounded_priority_queue", "autogenerated_element_samplers", ]] cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, diff --git a/yt/utilities/lib/alt_ray_tracers.pyx b/yt/utilities/lib/alt_ray_tracers.pyx index 1678747b51e..c7ef82f8720 100644 --- a/yt/utilities/lib/alt_ray_tracers.pyx +++ b/yt/utilities/lib/alt_ray_tracers.pyx @@ -1,3 +1,4 @@ +# distutils: libraries = STD_LIBS """ diff --git a/yt/utilities/lib/element_mappings.pyx b/yt/utilities/lib/element_mappings.pyx index d88c83c2a57..26ef449a2cc 100644 --- a/yt/utilities/lib/element_mappings.pyx +++ b/yt/utilities/lib/element_mappings.pyx @@ -1,3 +1,4 @@ +# distutils: libraries = STD_LIBS """ This file contains coordinate mappings between physical coordinates and those defined on unit elements, as well as doing the corresponding intracell diff --git a/yt/utilities/lib/primitives.pyx b/yt/utilities/lib/primitives.pyx index 289a30ad179..42b7f8ac85d 100644 --- a/yt/utilities/lib/primitives.pyx +++ b/yt/utilities/lib/primitives.pyx @@ -1,3 +1,4 @@ +# distutils: libraries = STD_LIBS """ This file contains definitions of the various primitives that can be used From 5c3c458ac8efb185966f511375bdf87681a0e502 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 11:58:46 -0500 Subject: [PATCH 06/17] Continuing to convert files that need arguments --- setup.py | 74 +++---------------- yt/frontends/ramses/io_utils.pyx | 2 + yt/utilities/cython_fortran_utils.pyx | 1 + .../lib/bounding_volume_hierarchy.pyx | 4 + yt/utilities/lib/contour_finding.pyx | 2 + yt/utilities/lib/cyoctree.pyx | 5 ++ yt/utilities/lib/ewah_bool_wrap.pyx | 2 + yt/utilities/lib/geometry_utils.pyx | 3 + yt/utilities/lib/grid_traversal.pyx | 3 + yt/utilities/lib/marching_cubes.pyx | 3 + yt/utilities/lib/misc_utilities.pyx | 3 + yt/utilities/lib/partitioned_grid.pyx | 3 + 12 files changed, 43 insertions(+), 62 deletions(-) diff --git a/setup.py b/setup.py index da56aa8682c..9e00ddba13d 100644 --- a/setup.py +++ b/setup.py @@ -86,22 +86,13 @@ def _compile( from Cython.Build import cythonize cythonize_aliases = {'LIB_DIR': 'yt/utilities/lib/', 'LIB_DIR_EWAH': ['yt/utilities/lib/', 'yt/utilities/lib/ewahboolarray/'], - 'STD_LIBS': std_libs} + 'LIB_DIR_GEOM': ['yt/utilities/lib/', 'yt/geometry/'], + 'STD_LIBS': std_libs, + 'OMP_ARGS': omp_args, + 'FIXED_INTERP': 'yt/utilities/lib/fixed_interpolator.c'} cython_extensions = [] cython_extensions += [ - Extension("yt.utilities.lib.bounding_volume_hierarchy", - ["yt/utilities/lib/bounding_volume_hierarchy.pyx"], - include_dirs=["yt/utilities/lib/"], - extra_compile_args=omp_args, - extra_link_args=omp_args, - libraries=std_libs, - depends=["yt/utilities/lib/mesh_triangulation.h"]), - Extension("yt.utilities.lib.contour_finding", - ["yt/utilities/lib/contour_finding.pyx"], - include_dirs=["yt/utilities/lib/", - "yt/geometry/"], - libraries=std_libs), Extension("yt.utilities.lib.cykdtree.kdtree", [ "yt/utilities/lib/cykdtree/kdtree.pyx", @@ -114,7 +105,7 @@ def _compile( ], libraries=std_libs, language="c++", - extra_compile_arg=["-std=c++03"]), + extra_compile_args=["-std=c++03"]), Extension("yt.utilities.lib.cykdtree.utils", [ "yt/utilities/lib/cykdtree/utils.pyx", @@ -123,18 +114,7 @@ def _compile( depends=["yt/utilities/lib/cykdtree/c_utils.hpp"], libraries=std_libs, language="c++", - extra_compile_arg=["-std=c++03"]), - Extension("yt.utilities.lib.geometry_utils", - ["yt/utilities/lib/geometry_utils.pyx"], - extra_compile_args=omp_args, - extra_link_args=omp_args, - libraries=std_libs), - Extension("yt.utilities.lib.marching_cubes", - ["yt/utilities/lib/marching_cubes.pyx", - "yt/utilities/lib/fixed_interpolator.c"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs, - depends=["yt/utilities/lib/fixed_interpolator.h"]), + extra_compile_args=["-std=c++03"]), Extension("yt.utilities.lib.pixelization_routines", ["yt/utilities/lib/pixelization_routines.pyx", "yt/utilities/lib/pixelization_constants.c"], @@ -144,28 +124,11 @@ def _compile( language='c++', libraries=std_libs, depends=["yt/utilities/lib/pixelization_constants.h"]), - Extension("yt.utilities.lib.cyoctree", - ["yt/utilities/lib/cyoctree.pyx"], - extra_compile_args=omp_args, - extra_link_args=omp_args, - libraries=std_libs, - language='c++'), Extension("yt.utilities.lib.origami", ["yt/utilities/lib/origami.pyx", "yt/utilities/lib/origami_tags.c"], include_dirs=["yt/utilities/lib/"], depends=["yt/utilities/lib/origami_tags.h"]), - Extension("yt.utilities.lib.grid_traversal", - ["yt/utilities/lib/grid_traversal.pyx", - "yt/utilities/lib/fixed_interpolator.c"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs, - depends=["yt/utilities/lib/fixed_interpolator.h"]), - Extension("yt.utilities.lib.ewah_bool_wrap", - ["yt/utilities/lib/ewah_bool_wrap.pyx"], - include_dirs=["yt/utilities/lib/", - "yt/utilities/lib/ewahboolarray"], - language="c++"), Extension("yt.utilities.lib.image_samplers", ["yt/utilities/lib/image_samplers.pyx", "yt/utilities/lib/fixed_interpolator.c"], @@ -174,27 +137,11 @@ def _compile( extra_compile_args=omp_args, extra_link_args=omp_args, depends=["yt/utilities/lib/fixed_interpolator.h"]), - Extension("yt.utilities.lib.partitioned_grid", - ["yt/utilities/lib/partitioned_grid.pyx", - "yt/utilities/lib/fixed_interpolator.c"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs, - depends=["yt/utilities/lib/fixed_interpolator.h"]), - Extension("yt.utilities.lib.misc_utilities", - ["yt/utilities/lib/misc_utilities.pyx"], - extra_compile_args=omp_args, - extra_link_args=omp_args, - libraries=std_libs), - Extension("yt.frontends.ramses.io_utils", - ["yt/frontends/ramses/io_utils.pyx"], - include_dirs=["yt/utilities/lib"], - libraries=std_libs), - Extension("yt.utilities.cython_fortran_utils", - ["yt/utilities/cython_fortran_utils.pyx"], - libraries=std_libs), ] -lib_exts = ["yt/geometry/*.pyx"] + ["yt/utilities/lib/{}.pyx".format(_) for _ in [ +lib_exts = ["yt/geometry/*.pyx", "yt/utilities/cython_fortran_utils.pyx", + "yt/frontends/ramses/io_utils.pyx"] +lib_exts += ["yt/utilities/lib/{}.pyx".format(_) for _ in [ "particle_mesh_operations", "depth_first_octree", "fortran_reader", "interpolators", "basic_octree", "image_utilities", "points_in_volume", "quad_tree", "mesh_utilities", @@ -203,6 +150,9 @@ def _compile( "bitarray", "fnv_hash", "mesh_triangulation", "particle_kdtree_tools", "primitives", "cosmology_time", "element_mappings", "alt_ray_tracers", "bounded_priority_queue", "autogenerated_element_samplers", + "geometry_utils", "bounding_volume_hierarchy", "contour_finding", + "marching_cubes", "cyoctree", "grid_traversal", "ewah_bool_wrap", + "partitioned_grid", "misc_utilities" ]] cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, diff --git a/yt/frontends/ramses/io_utils.pyx b/yt/frontends/ramses/io_utils.pyx index 160cdb62b39..3e73e6bfed3 100644 --- a/yt/frontends/ramses/io_utils.pyx +++ b/yt/frontends/ramses/io_utils.pyx @@ -1,3 +1,5 @@ +# distutils: libraries = STD_LIBS +# distutils: include_dirs = LIB_DIR cimport cython cimport numpy as np import numpy as np diff --git a/yt/utilities/cython_fortran_utils.pyx b/yt/utilities/cython_fortran_utils.pyx index f52c9a8773e..e4aa9d285d3 100644 --- a/yt/utilities/cython_fortran_utils.pyx +++ b/yt/utilities/cython_fortran_utils.pyx @@ -1,3 +1,4 @@ +# distutils: libraries = STD_LIBS cimport numpy as np import numpy as np import cython diff --git a/yt/utilities/lib/bounding_volume_hierarchy.pyx b/yt/utilities/lib/bounding_volume_hierarchy.pyx index 7743dde7452..fe40961fd92 100644 --- a/yt/utilities/lib/bounding_volume_hierarchy.pyx +++ b/yt/utilities/lib/bounding_volume_hierarchy.pyx @@ -1,3 +1,7 @@ +# distutils: libraries = STD_LIBS +# distutils: include_dirs = LIB_DIR +# distutils: extra_compile_args = OMP_ARGS +# distutils: extra_link_args = OMP_ARGS cimport cython import numpy as np cimport numpy as np diff --git a/yt/utilities/lib/contour_finding.pyx b/yt/utilities/lib/contour_finding.pyx index bf3ab59e90f..948fffac563 100644 --- a/yt/utilities/lib/contour_finding.pyx +++ b/yt/utilities/lib/contour_finding.pyx @@ -1,3 +1,5 @@ +# distutils: libraries = STD_LIBS +# distutils: include_dirs = LIB_DIR_GEOM """ A two-pass contour finding algorithm diff --git a/yt/utilities/lib/cyoctree.pyx b/yt/utilities/lib/cyoctree.pyx index b2bf5b14d20..8dc02c599a1 100644 --- a/yt/utilities/lib/cyoctree.pyx +++ b/yt/utilities/lib/cyoctree.pyx @@ -1,3 +1,8 @@ +# distutils: libraries = STD_LIBS +# distutils: extra_link_args = OMP_ARGS +# distutils: extra_compile_args = OMP_ARGS +# distutils: include_dirs = LIB_DIR +# distutils: language = c++ """ CyOctree building, loading and refining routines diff --git a/yt/utilities/lib/ewah_bool_wrap.pyx b/yt/utilities/lib/ewah_bool_wrap.pyx index b4ced2c87bd..14fba1a0bcf 100644 --- a/yt/utilities/lib/ewah_bool_wrap.pyx +++ b/yt/utilities/lib/ewah_bool_wrap.pyx @@ -1,3 +1,5 @@ +# distutils: language = c++ +# distutils: include_dirs = LIB_DIR_EWAH """ Wrapper for EWAH Bool Array: https://github.com/lemire/EWAHBoolArray diff --git a/yt/utilities/lib/geometry_utils.pyx b/yt/utilities/lib/geometry_utils.pyx index 5e94a7975ea..bf468d5f984 100644 --- a/yt/utilities/lib/geometry_utils.pyx +++ b/yt/utilities/lib/geometry_utils.pyx @@ -1,3 +1,6 @@ +# distutils: libraries = STD_LIBS +# distutils: extra_compile_args = OMP_ARGS +# distutils: extra_link_args = OMP_ARGS """ Simple integrators for the radiative transfer equation diff --git a/yt/utilities/lib/grid_traversal.pyx b/yt/utilities/lib/grid_traversal.pyx index 10d8e354fb8..c80678ce0a2 100644 --- a/yt/utilities/lib/grid_traversal.pyx +++ b/yt/utilities/lib/grid_traversal.pyx @@ -1,3 +1,6 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS +# distutils: sources = FIXED_INTERP """ Simple integrators for the radiative transfer equation diff --git a/yt/utilities/lib/marching_cubes.pyx b/yt/utilities/lib/marching_cubes.pyx index ce18bbbe8c6..a23db6d3561 100644 --- a/yt/utilities/lib/marching_cubes.pyx +++ b/yt/utilities/lib/marching_cubes.pyx @@ -1,3 +1,6 @@ +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS +# distutils: sources = FIXED_INTERP """ Marching cubes implementation diff --git a/yt/utilities/lib/misc_utilities.pyx b/yt/utilities/lib/misc_utilities.pyx index 701dec9bb38..52af0e6522d 100644 --- a/yt/utilities/lib/misc_utilities.pyx +++ b/yt/utilities/lib/misc_utilities.pyx @@ -1,3 +1,6 @@ +# distutils: libraries = STD_LIBS +# distutils: extra_compile_args = OMP_ARGS +# distutils: extra_link_args = OMP_ARGS """ Simple utilities that don't fit anywhere else diff --git a/yt/utilities/lib/partitioned_grid.pyx b/yt/utilities/lib/partitioned_grid.pyx index 12502b108fb..7137fca2784 100644 --- a/yt/utilities/lib/partitioned_grid.pyx +++ b/yt/utilities/lib/partitioned_grid.pyx @@ -1,3 +1,6 @@ +# distutils: sources = FIXED_INTERP +# distutils: include_dirs = LIB_DIR +# distutils: libraries = STD_LIBS """ Image sampler definitions From ff9f157d874c6eba5a5b3f4174049e5c19a820dd Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 15:04:58 -0500 Subject: [PATCH 07/17] Almost done converting to cythonize --- setup.py | 75 +++++----------------- yt/frontends/artio/_artio_caller.pyx | 2 + yt/utilities/lib/cykdtree/kdtree.pyx | 6 ++ yt/utilities/lib/cykdtree/utils.pyx | 5 ++ yt/utilities/lib/image_samplers.pyx | 5 ++ yt/utilities/lib/origami.pyx | 3 + yt/utilities/lib/pixelization_routines.pyx | 6 ++ 7 files changed, 42 insertions(+), 60 deletions(-) diff --git a/setup.py b/setup.py index 9e00ddba13d..b3e328259ff 100644 --- a/setup.py +++ b/setup.py @@ -87,60 +87,24 @@ def _compile( cythonize_aliases = {'LIB_DIR': 'yt/utilities/lib/', 'LIB_DIR_EWAH': ['yt/utilities/lib/', 'yt/utilities/lib/ewahboolarray/'], 'LIB_DIR_GEOM': ['yt/utilities/lib/', 'yt/geometry/'], + 'LIB_DIR_GEOM_ARTIO': ['yt/utilities/lib/', 'yt/geometry/', + 'yt/frontends/artio/artio_headers/'], 'STD_LIBS': std_libs, 'OMP_ARGS': omp_args, - 'FIXED_INTERP': 'yt/utilities/lib/fixed_interpolator.c'} + 'FIXED_INTERP': 'yt/utilities/lib/fixed_interpolator.c', + 'ARTIO_SOURCE': glob.glob("yt/frontends/artio/artio_headers/*.c"), +} cython_extensions = [] -cython_extensions += [ - Extension("yt.utilities.lib.cykdtree.kdtree", - [ - "yt/utilities/lib/cykdtree/kdtree.pyx", - "yt/utilities/lib/cykdtree/c_kdtree.cpp", - "yt/utilities/lib/cykdtree/c_utils.cpp", - ], - depends=[ - "yt/utilities/lib/cykdtree/c_kdtree.hpp", - "yt/utilities/lib/cykdtree/c_utils.hpp", - ], - libraries=std_libs, - language="c++", - extra_compile_args=["-std=c++03"]), - Extension("yt.utilities.lib.cykdtree.utils", - [ - "yt/utilities/lib/cykdtree/utils.pyx", - "yt/utilities/lib/cykdtree/c_utils.cpp", - ], - depends=["yt/utilities/lib/cykdtree/c_utils.hpp"], - libraries=std_libs, - language="c++", - extra_compile_args=["-std=c++03"]), - Extension("yt.utilities.lib.pixelization_routines", - ["yt/utilities/lib/pixelization_routines.pyx", - "yt/utilities/lib/pixelization_constants.c"], - include_dirs=["yt/utilities/lib/"], - extra_compile_args=omp_args, - extra_link_args=omp_args, - language='c++', - libraries=std_libs, - depends=["yt/utilities/lib/pixelization_constants.h"]), - Extension("yt.utilities.lib.origami", - ["yt/utilities/lib/origami.pyx", - "yt/utilities/lib/origami_tags.c"], - include_dirs=["yt/utilities/lib/"], - depends=["yt/utilities/lib/origami_tags.h"]), - Extension("yt.utilities.lib.image_samplers", - ["yt/utilities/lib/image_samplers.pyx", - "yt/utilities/lib/fixed_interpolator.c"], - include_dirs=["yt/utilities/lib/"], - libraries=std_libs, - extra_compile_args=omp_args, - extra_link_args=omp_args, - depends=["yt/utilities/lib/fixed_interpolator.h"]), + +lib_exts = ["yt/geometry/*.pyx", + "yt/utilities/cython_fortran_utils.pyx", + "yt/frontends/ramses/io_utils.pyx", + "yt/utilities/lib/cykdtree/kdtree.pyx", + "yt/utilities/lib/cykdtree/utils.pyx", + "yt/frontends/artio/_artio_caller.pyx", ] -lib_exts = ["yt/geometry/*.pyx", "yt/utilities/cython_fortran_utils.pyx", - "yt/frontends/ramses/io_utils.pyx"] lib_exts += ["yt/utilities/lib/{}.pyx".format(_) for _ in [ "particle_mesh_operations", "depth_first_octree", "fortran_reader", "interpolators", "basic_octree", "image_utilities", @@ -152,22 +116,13 @@ def _compile( "bounded_priority_queue", "autogenerated_element_samplers", "geometry_utils", "bounding_volume_hierarchy", "contour_finding", "marching_cubes", "cyoctree", "grid_traversal", "ewah_bool_wrap", - "partitioned_grid", "misc_utilities" + "partitioned_grid", "misc_utilities", "pixelization_routines", + "image_samplers", "origami" ]] cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, language_level = 2) -extensions = [ - Extension("yt.frontends.artio._artio_caller", - ["yt/frontends/artio/_artio_caller.pyx"] + - glob.glob("yt/frontends/artio/artio_headers/*.c"), - include_dirs=["yt/frontends/artio/artio_headers/", - "yt/geometry/", - "yt/utilities/lib/"], - depends=glob.glob("yt/frontends/artio/artio_headers/*.c")), -] - # EMBREE if check_for_pyembree() is not None: embree_extensions = [ @@ -340,6 +295,6 @@ def run(self): license="BSD 3-Clause", zip_safe=False, scripts=["scripts/iyt"], - ext_modules=cython_extensions + extensions, + ext_modules=cython_extensions, python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*' ) diff --git a/yt/frontends/artio/_artio_caller.pyx b/yt/frontends/artio/_artio_caller.pyx index 5a69f57495f..15eb124376f 100644 --- a/yt/frontends/artio/_artio_caller.pyx +++ b/yt/frontends/artio/_artio_caller.pyx @@ -1,3 +1,5 @@ +# distutils: sources = ARTIO_SOURCE +# distutils: include_dirs = LIB_DIR_GEOM_ARTIO cimport cython import numpy as np cimport numpy as np diff --git a/yt/utilities/lib/cykdtree/kdtree.pyx b/yt/utilities/lib/cykdtree/kdtree.pyx index 928cca6d95c..383e901d2ca 100644 --- a/yt/utilities/lib/cykdtree/kdtree.pyx +++ b/yt/utilities/lib/cykdtree/kdtree.pyx @@ -1,3 +1,9 @@ +# distutils: libraries = STD_LIBS +# Note that we used to include the empty c_kdtree.cpp file, but that seems to break cythonize. +# distutils: sources = yt/utilities/lib/cykdtree/c_utils.cpp +# distutils: depends = yt/utilities/lib/cykdtree/c_kdtree.hpp, yt/utilities/lib/cykdtree/c_utils.hpp +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++03 import cython import numpy as np cimport numpy as np diff --git a/yt/utilities/lib/cykdtree/utils.pyx b/yt/utilities/lib/cykdtree/utils.pyx index 4d3696e0115..d5dba21b9db 100644 --- a/yt/utilities/lib/cykdtree/utils.pyx +++ b/yt/utilities/lib/cykdtree/utils.pyx @@ -1,3 +1,8 @@ +# distutils: libraries = STD_LIBS +# distutils: sources = yt/utilities/lib/cykdtree/c_utils.cpp +# distutils: depends = yt/utilities/lib/cykdtree/c_utils.hpp +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++03 import numpy as np cimport numpy as np cimport cython diff --git a/yt/utilities/lib/image_samplers.pyx b/yt/utilities/lib/image_samplers.pyx index 214e2cd1d38..28bbb19e62f 100644 --- a/yt/utilities/lib/image_samplers.pyx +++ b/yt/utilities/lib/image_samplers.pyx @@ -1,3 +1,8 @@ +# distutils: include_dirs = LIB_DIR +# distutils: extra_compile_args = OMP_ARGS +# distutils: extra_link_args = OMP_ARGS +# distutils: libraries = STD_LIBS +# distutils: sources = FIXED_INTERP """ Image sampler definitions diff --git a/yt/utilities/lib/origami.pyx b/yt/utilities/lib/origami.pyx index 0510eec4846..3631064e354 100644 --- a/yt/utilities/lib/origami.pyx +++ b/yt/utilities/lib/origami.pyx @@ -1,3 +1,6 @@ +# distutils: sources = yt/utilities/lib/origami_tags.c +# distutils: include_dirs = LIB_DIR +# distutils: depends = yt/utilities/lib/origami_tags.h """ This calls the ORIGAMI routines diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index aecd71f2289..ee50902de2f 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1,3 +1,9 @@ +# distutils: include_dirs = LIB_DIR +# distutils: extra_compile_args = OMP_ARGS +# distutils: extra_link_args = OMP_ARGS +# distutils: language = c++ +# distutils: libraries = STD_LIBS +# distutils: sources = yt/utilities/lib/pixelization_constants.c """ Pixelization routines From f744424a6a795796d14f464e73613b9d21bac021 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 15:49:49 -0500 Subject: [PATCH 08/17] Attempting to convert embree imports --- setup.py | 30 +++++++------------ .../{ => embree_mesh}/mesh_construction.pxd | 0 .../{ => embree_mesh}/mesh_construction.pyx | 4 +++ .../{ => embree_mesh}/mesh_intersection.pxd | 0 .../{ => embree_mesh}/mesh_intersection.pyx | 4 +++ .../lib/{ => embree_mesh}/mesh_samplers.pxd | 0 .../lib/{ => embree_mesh}/mesh_samplers.pyx | 4 +++ .../lib/{ => embree_mesh}/mesh_traversal.pxd | 0 .../lib/{ => embree_mesh}/mesh_traversal.pyx | 6 +++- .../lib/{ => embree_mesh}/vec3_ops.pxd | 0 .../volume_rendering/render_source.py | 4 +-- yt/visualization/volume_rendering/utils.py | 2 +- 12 files changed, 30 insertions(+), 24 deletions(-) rename yt/utilities/lib/{ => embree_mesh}/mesh_construction.pxd (100%) rename yt/utilities/lib/{ => embree_mesh}/mesh_construction.pyx (98%) rename yt/utilities/lib/{ => embree_mesh}/mesh_intersection.pxd (100%) rename yt/utilities/lib/{ => embree_mesh}/mesh_intersection.pyx (96%) rename yt/utilities/lib/{ => embree_mesh}/mesh_samplers.pxd (100%) rename yt/utilities/lib/{ => embree_mesh}/mesh_samplers.pyx (98%) rename yt/utilities/lib/{ => embree_mesh}/mesh_traversal.pxd (100%) rename yt/utilities/lib/{ => embree_mesh}/mesh_traversal.pyx (93%) rename yt/utilities/lib/{ => embree_mesh}/vec3_ops.pxd (100%) diff --git a/setup.py b/setup.py index b3e328259ff..6504d3b5bfe 100644 --- a/setup.py +++ b/setup.py @@ -125,21 +125,10 @@ def _compile( # EMBREE if check_for_pyembree() is not None: - embree_extensions = [ - Extension("yt.utilities.lib.mesh_construction", - ["yt/utilities/lib/mesh_construction.pyx"], - depends=["yt/utilities/lib/mesh_triangulation.h"]), - Extension("yt.utilities.lib.mesh_traversal", - ["yt/utilities/lib/mesh_traversal.pyx"]), - Extension("yt.utilities.lib.mesh_samplers", - ["yt/utilities/lib/mesh_samplers.pyx"]), - Extension("yt.utilities.lib.mesh_intersection", - ["yt/utilities/lib/mesh_intersection.pyx"]), - ] embree_prefix = os.path.abspath(read_embree_location()) - embree_inc_dir = [os.path.join(embree_prefix, 'include')] - embree_lib_dir = [os.path.join(embree_prefix, 'lib')] + embree_inc_dir = os.path.join(embree_prefix, 'include') + embree_lib_dir = os.path.join(embree_prefix, 'lib') if in_conda_env(): conda_basedir = os.path.dirname(os.path.dirname(sys.executable)) embree_inc_dir.append(os.path.join(conda_basedir, 'include')) @@ -150,14 +139,15 @@ def _compile( else: embree_lib_name = "embree" - for ext in embree_extensions: - ext.include_dirs += embree_inc_dir - ext.library_dirs += embree_lib_dir - ext.language = "c++" - ext.libraries += std_libs - ext.libraries += [embree_lib_name] + embree_aliases = cythonize_aliases.copy() + embree_aliases['EMBREE_INC_DIR'] = ['yt/utilities/lib/', embree_inc_dir] + embree_aliases['EMBREE_LIB_DIR'] = [embree_lib_dir] + embree_aliases['EMBREE_LIBS'] = std_libs + [embree_lib_name] - cython_extensions += embree_extensions + cython_extensions += cythonize("yt/utilities/lib/embree_mesh/*.pyx", + aliases = embree_aliases, + language_level = 2 + ) if os.environ.get("GPERFTOOLS", "no").upper() != "NO": gpd = os.environ["GPERFTOOLS"] diff --git a/yt/utilities/lib/mesh_construction.pxd b/yt/utilities/lib/embree_mesh/mesh_construction.pxd similarity index 100% rename from yt/utilities/lib/mesh_construction.pxd rename to yt/utilities/lib/embree_mesh/mesh_construction.pxd diff --git a/yt/utilities/lib/mesh_construction.pyx b/yt/utilities/lib/embree_mesh/mesh_construction.pyx similarity index 98% rename from yt/utilities/lib/mesh_construction.pyx rename to yt/utilities/lib/embree_mesh/mesh_construction.pyx index 47db9f4f6c0..f883e0d73b8 100644 --- a/yt/utilities/lib/mesh_construction.pyx +++ b/yt/utilities/lib/embree_mesh/mesh_construction.pyx @@ -1,3 +1,7 @@ +# distutils: include_dirs = EMBREE_INC_DIR +# distutils: library_dirs = EMBREE_LIB_DIR +# distutils: libraries = EMBREE_LIBS +# distutils: language = c++ """ This file contains the ElementMesh classes, which represent the target that the rays will be cast at when rendering finite element data. This class handles diff --git a/yt/utilities/lib/mesh_intersection.pxd b/yt/utilities/lib/embree_mesh/mesh_intersection.pxd similarity index 100% rename from yt/utilities/lib/mesh_intersection.pxd rename to yt/utilities/lib/embree_mesh/mesh_intersection.pxd diff --git a/yt/utilities/lib/mesh_intersection.pyx b/yt/utilities/lib/embree_mesh/mesh_intersection.pyx similarity index 96% rename from yt/utilities/lib/mesh_intersection.pyx rename to yt/utilities/lib/embree_mesh/mesh_intersection.pyx index 28d501fcc66..33a4075d521 100644 --- a/yt/utilities/lib/mesh_intersection.pyx +++ b/yt/utilities/lib/embree_mesh/mesh_intersection.pyx @@ -1,3 +1,7 @@ +# distutils: include_dirs = EMBREE_INC_DIR +# distutils: library_dirs = EMBREE_LIB_DIR +# distutils: libraries = EMBREE_LIBS +# distutils: language = c++ """ This file contains functions used for performing ray-tracing with Embree for 2nd-order Lagrange Elements. diff --git a/yt/utilities/lib/mesh_samplers.pxd b/yt/utilities/lib/embree_mesh/mesh_samplers.pxd similarity index 100% rename from yt/utilities/lib/mesh_samplers.pxd rename to yt/utilities/lib/embree_mesh/mesh_samplers.pxd diff --git a/yt/utilities/lib/mesh_samplers.pyx b/yt/utilities/lib/embree_mesh/mesh_samplers.pyx similarity index 98% rename from yt/utilities/lib/mesh_samplers.pyx rename to yt/utilities/lib/embree_mesh/mesh_samplers.pyx index 9381f7d3483..53557d85aa5 100644 --- a/yt/utilities/lib/mesh_samplers.pyx +++ b/yt/utilities/lib/embree_mesh/mesh_samplers.pyx @@ -1,3 +1,7 @@ +# distutils: include_dirs = EMBREE_INC_DIR +# distutils: library_dirs = EMBREE_LIB_DIR +# distutils: libraries = EMBREE_LIBS +# distutils: language = c++ """ This file contains functions that sample a surface mesh at the point hit by a ray. These can be used with pyembree in the form of "filter feedback functions." diff --git a/yt/utilities/lib/mesh_traversal.pxd b/yt/utilities/lib/embree_mesh/mesh_traversal.pxd similarity index 100% rename from yt/utilities/lib/mesh_traversal.pxd rename to yt/utilities/lib/embree_mesh/mesh_traversal.pxd diff --git a/yt/utilities/lib/mesh_traversal.pyx b/yt/utilities/lib/embree_mesh/mesh_traversal.pyx similarity index 93% rename from yt/utilities/lib/mesh_traversal.pyx rename to yt/utilities/lib/embree_mesh/mesh_traversal.pyx index 11839be98bc..8236ea13d23 100644 --- a/yt/utilities/lib/mesh_traversal.pyx +++ b/yt/utilities/lib/embree_mesh/mesh_traversal.pyx @@ -1,3 +1,7 @@ +# distutils: include_dirs = EMBREE_INC_DIR +# distutils: library_dirs = EMBREE_LIB_DIR +# distutils: libraries = EMBREE_LIBS +# distutils: language = c++ """ This file contains the MeshSampler classes, which handles casting rays at a mesh source using either pyembree or the cython ray caster. @@ -14,7 +18,7 @@ cimport pyembree.rtcore as rtc cimport pyembree.rtcore_ray as rtcr cimport pyembree.rtcore_geometry as rtcg cimport pyembree.rtcore_scene as rtcs -from .image_samplers cimport \ +from yt.utilities.lib.image_samplers cimport \ ImageSampler from cython.parallel import prange, parallel, threadid from yt.visualization.image_writer import apply_colormap diff --git a/yt/utilities/lib/vec3_ops.pxd b/yt/utilities/lib/embree_mesh/vec3_ops.pxd similarity index 100% rename from yt/utilities/lib/vec3_ops.pxd rename to yt/utilities/lib/embree_mesh/vec3_ops.pxd diff --git a/yt/visualization/volume_rendering/render_source.py b/yt/visualization/volume_rendering/render_source.py index a31bcb54dc9..86e48401404 100644 --- a/yt/visualization/volume_rendering/render_source.py +++ b/yt/visualization/volume_rendering/render_source.py @@ -21,13 +21,13 @@ from yt.utilities.on_demand_imports import NotAModule try: - from yt.utilities.lib import mesh_traversal + from yt.utilities.lib.embree_mesh import mesh_traversal # Catch ValueError in case size of objects in Cython change except (ImportError, ValueError): mesh_traversal = NotAModule("pyembree") ytcfg["yt", "ray_tracing_engine"] = "yt" try: - from yt.utilities.lib import mesh_construction + from yt.utilities.lib.embree_mesh import mesh_construction # Catch ValueError in case size of objects in Cython change except (ImportError, ValueError): mesh_construction = NotAModule("pyembree") diff --git a/yt/visualization/volume_rendering/utils.py b/yt/visualization/volume_rendering/utils.py index 29ea21e0818..d7bb1dabec0 100644 --- a/yt/visualization/volume_rendering/utils.py +++ b/yt/visualization/volume_rendering/utils.py @@ -8,7 +8,7 @@ from yt.utilities.on_demand_imports import NotAModule try: - from yt.utilities.lib import mesh_traversal + from yt.utilities.lib.embree_mesh import mesh_traversal # Catch ValueError in case size of objects in Cython change except (ImportError, ValueError): mesh_traversal = NotAModule("pyembree") From 53d28f8ce812286b0039763178174a6d31b6a6c9 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 16:02:34 -0500 Subject: [PATCH 09/17] Go to globs everywhere --- setup.py | 34 ++++++------------------ yt/utilities/lib/embree_mesh/__init__.py | 0 2 files changed, 8 insertions(+), 26 deletions(-) create mode 100644 yt/utilities/lib/embree_mesh/__init__.py diff --git a/setup.py b/setup.py index 6504d3b5bfe..a11d7f11817 100644 --- a/setup.py +++ b/setup.py @@ -103,26 +103,9 @@ def _compile( "yt/utilities/lib/cykdtree/kdtree.pyx", "yt/utilities/lib/cykdtree/utils.pyx", "yt/frontends/artio/_artio_caller.pyx", + "yt/utilities/lib/*.pyx", ] -lib_exts += ["yt/utilities/lib/{}.pyx".format(_) for _ in [ - "particle_mesh_operations", "depth_first_octree", "fortran_reader", - "interpolators", "basic_octree", "image_utilities", - "points_in_volume", "quad_tree", "mesh_utilities", - "amr_kdtools", "lenses", "distance_queue", "allocation_container", - "write_array", "ragged_arrays", "line_integral_convolution", - "bitarray", "fnv_hash", "mesh_triangulation", "particle_kdtree_tools", - "primitives", "cosmology_time", "element_mappings", "alt_ray_tracers", - "bounded_priority_queue", "autogenerated_element_samplers", - "geometry_utils", "bounding_volume_hierarchy", "contour_finding", - "marching_cubes", "cyoctree", "grid_traversal", "ewah_bool_wrap", - "partitioned_grid", "misc_utilities", "pixelization_routines", - "image_samplers", "origami" -]] - -cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, - language_level = 2) - # EMBREE if check_for_pyembree() is not None: @@ -139,15 +122,14 @@ def _compile( else: embree_lib_name = "embree" - embree_aliases = cythonize_aliases.copy() - embree_aliases['EMBREE_INC_DIR'] = ['yt/utilities/lib/', embree_inc_dir] - embree_aliases['EMBREE_LIB_DIR'] = [embree_lib_dir] - embree_aliases['EMBREE_LIBS'] = std_libs + [embree_lib_name] + cythonize_aliases['EMBREE_INC_DIR'] = ['yt/utilities/lib/', embree_inc_dir] + cythonize_aliases['EMBREE_LIB_DIR'] = [embree_lib_dir] + cythonize_aliases['EMBREE_LIBS'] = std_libs + [embree_lib_name] + lib_exts += ['yt/utilities/lib/embree_mesh/*.pyx'] + +cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, + language_level = 2) - cython_extensions += cythonize("yt/utilities/lib/embree_mesh/*.pyx", - aliases = embree_aliases, - language_level = 2 - ) if os.environ.get("GPERFTOOLS", "no").upper() != "NO": gpd = os.environ["GPERFTOOLS"] diff --git a/yt/utilities/lib/embree_mesh/__init__.py b/yt/utilities/lib/embree_mesh/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 27fcc58f69a7238c43437460dff6be83e7a9ab71 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 17:44:29 -0500 Subject: [PATCH 10/17] Switch back to the overridden build_ext --- setup.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index a11d7f11817..aae19d33a98 100644 --- a/setup.py +++ b/setup.py @@ -95,8 +95,6 @@ def _compile( 'ARTIO_SOURCE': glob.glob("yt/frontends/artio/artio_headers/*.c"), } -cython_extensions = [] - lib_exts = ["yt/geometry/*.pyx", "yt/utilities/cython_fortran_utils.pyx", "yt/frontends/ramses/io_utils.pyx", @@ -127,10 +125,6 @@ def _compile( cythonize_aliases['EMBREE_LIBS'] = std_libs + [embree_lib_name] lib_exts += ['yt/utilities/lib/embree_mesh/*.pyx'] -cython_extensions += cythonize(lib_exts, aliases = cythonize_aliases, - language_level = 2) - - if os.environ.get("GPERFTOOLS", "no").upper() != "NO": gpd = os.environ["GPERFTOOLS"] idir = os.path.join(gpd, "include") @@ -168,8 +162,10 @@ def finalize_options(self): package manager for your python environment.""" % numpy.__version__) from Cython.Build import cythonize + # Override the list of extension modules self.distribution.ext_modules[:] = cythonize( - self.distribution.ext_modules, + lib_exts, + aliases = cythonize_aliases, compiler_directives={'language_level': 2}, nthreads=_get_cpu_count(), ) @@ -204,7 +200,8 @@ def run(self): # Make sure the compiled Cython files in the distribution are up-to-date from Cython.Build import cythonize cythonize( - cython_extensions, + lib_exts, + aliases = cythonize_aliases, compiler_directives={'language_level': 2}, nthreads=_get_cpu_count(), ) @@ -267,6 +264,6 @@ def run(self): license="BSD 3-Clause", zip_safe=False, scripts=["scripts/iyt"], - ext_modules=cython_extensions, + ext_modules=[], # !!! We override this inside build_ext above python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*' ) From 20838e65b2b05cbf6d6d1c68de9ec280f90027ab Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 30 Mar 2020 17:45:54 -0500 Subject: [PATCH 11/17] Only import Cython as needed --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index aae19d33a98..22437dd512e 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,6 @@ def _compile( else: std_libs = ["m"] -from Cython.Build import cythonize cythonize_aliases = {'LIB_DIR': 'yt/utilities/lib/', 'LIB_DIR_EWAH': ['yt/utilities/lib/', 'yt/utilities/lib/ewahboolarray/'], 'LIB_DIR_GEOM': ['yt/utilities/lib/', 'yt/geometry/'], From 25dd71b25edef5514992216a04daa7a2df87990b Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Tue, 31 Mar 2020 10:47:37 -0500 Subject: [PATCH 12/17] Move more of the extension construction to setupext.py --- setup.py | 148 ++++------------------------------------------------ setupext.py | 137 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 141 insertions(+), 144 deletions(-) diff --git a/setup.py b/setup.py index 22437dd512e..3cf40f2d1be 100644 --- a/setup.py +++ b/setup.py @@ -1,57 +1,23 @@ import os -import platform -from concurrent.futures import ThreadPoolExecutor as Pool import glob import sys -from sys import platform as _platform from setuptools import setup, find_packages -from setuptools.extension import Extension -from setuptools.command.build_ext import build_ext as _build_ext -from setuptools.command.sdist import sdist as _sdist from setupext import \ check_for_openmp, \ check_for_pyembree, \ - read_embree_location, \ - in_conda_env + install_ccompiler, \ + create_build_ext from distutils.version import LooseVersion -from distutils.ccompiler import CCompiler import pkg_resources - -def _get_cpu_count(): - if platform.system() != "Windows": - return os.cpu_count() - return 0 - - -def _compile( - self, sources, output_dir=None, macros=None, include_dirs=None, - debug=0, extra_preargs=None, extra_postargs=None, depends=None, -): - """Function to monkey-patch distutils.ccompiler.CCompiler""" - macros, objects, extra_postargs, pp_opts, build = self._setup_compile( - output_dir, macros, include_dirs, sources, depends, extra_postargs - ) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) - - # Return *all* object filenames, not just the ones we just built. - return objects - -CCompiler.compile = _compile - if sys.version_info < (3, 5): print("yt currently supports versions newer than Python 3.5") print("certain features may fail unexpectedly and silently with older " "versions.") sys.exit(1) +install_ccompiler() + try: distribute_ver = \ LooseVersion(pkg_resources.get_distribution("distribute").version) @@ -103,108 +69,12 @@ def _compile( "yt/utilities/lib/*.pyx", ] -# EMBREE -if check_for_pyembree() is not None: - - embree_prefix = os.path.abspath(read_embree_location()) - embree_inc_dir = os.path.join(embree_prefix, 'include') - embree_lib_dir = os.path.join(embree_prefix, 'lib') - if in_conda_env(): - conda_basedir = os.path.dirname(os.path.dirname(sys.executable)) - embree_inc_dir.append(os.path.join(conda_basedir, 'include')) - embree_lib_dir.append(os.path.join(conda_basedir, 'lib')) - - if _platform == "darwin": - embree_lib_name = "embree.2" - else: - embree_lib_name = "embree" - - cythonize_aliases['EMBREE_INC_DIR'] = ['yt/utilities/lib/', embree_inc_dir] - cythonize_aliases['EMBREE_LIB_DIR'] = [embree_lib_dir] - cythonize_aliases['EMBREE_LIBS'] = std_libs + [embree_lib_name] - lib_exts += ['yt/utilities/lib/embree_mesh/*.pyx'] - -if os.environ.get("GPERFTOOLS", "no").upper() != "NO": - gpd = os.environ["GPERFTOOLS"] - idir = os.path.join(gpd, "include") - ldir = os.path.join(gpd, "lib") - print(("INCLUDE AND LIB DIRS", idir, ldir)) - cython_extensions.append( - Extension("yt.utilities.lib.perftools_wrap", - ["yt/utilities/lib/perftools_wrap.pyx"], - libraries=["profiler"], - library_dirs=[ldir], - include_dirs=[idir])) - -class build_ext(_build_ext): - # subclass setuptools extension builder to avoid importing cython and numpy - # at top level in setup.py. See http://stackoverflow.com/a/21621689/1382869 - def finalize_options(self): - try: - import cython - import numpy - except ImportError: - raise ImportError( -"""Could not import cython or numpy. Building yt from source requires -cython and numpy to be installed. Please install these packages using -the appropriate package manager for your python environment.""") - if LooseVersion(cython.__version__) < LooseVersion('0.26.1'): - raise RuntimeError( -"""Building yt from source requires Cython 0.26.1 or newer but -Cython %s is installed. Please update Cython using the appropriate -package manager for your python environment.""" % - cython.__version__) - if LooseVersion(numpy.__version__) < LooseVersion('1.13.3'): - raise RuntimeError( -"""Building yt from source requires NumPy 1.13.3 or newer but -NumPy %s is installed. Please update NumPy using the appropriate -package manager for your python environment.""" % - numpy.__version__) - from Cython.Build import cythonize - # Override the list of extension modules - self.distribution.ext_modules[:] = cythonize( - lib_exts, - aliases = cythonize_aliases, - compiler_directives={'language_level': 2}, - nthreads=_get_cpu_count(), - ) - _build_ext.finalize_options(self) - # Prevent numpy from thinking it is still in its setup process - # see http://stackoverflow.com/a/21621493/1382869 - if isinstance(__builtins__, dict): - # sometimes this is a dict so we need to check for that - # https://docs.python.org/3/library/builtins.html - __builtins__["__NUMPY_SETUP__"] = False - else: - __builtins__.__NUMPY_SETUP__ = False - import numpy - self.include_dirs.append(numpy.get_include()) - - def build_extensions(self): - self.check_extensions_list(self.extensions) - - ncpus = _get_cpu_count() - if ncpus > 0: - with Pool(ncpus) as pool: - pool.map(self.build_extension, self.extensions) - else: - super().build_extensions() - +embree_libs, embree_aliases = check_for_pyembree(std_libs) +cythonize_aliases.update(embree_aliases) +lib_exts += embree_libs -class sdist(_sdist): - # subclass setuptools source distribution builder to ensure cython - # generated C files are included in source distribution. - # See http://stackoverflow.com/a/18418524/1382869 - def run(self): - # Make sure the compiled Cython files in the distribution are up-to-date - from Cython.Build import cythonize - cythonize( - lib_exts, - aliases = cythonize_aliases, - compiler_directives={'language_level': 2}, - nthreads=_get_cpu_count(), - ) - _sdist.run(self) +# This overrides using lib_exts, so it has to happen after lib_exts is fully defined +build_ext, sdist = create_build_ext(lib_exts, cythonize_aliases) setup( name="yt", diff --git a/setupext.py b/setupext.py index 92c456babe0..759142b25d3 100644 --- a/setupext.py +++ b/setupext.py @@ -1,17 +1,23 @@ +import os +import platform import contextlib import glob -import os import shutil import subprocess import sys +from sys import platform as _platform import tempfile +from concurrent.futures import ThreadPoolExecutor as Pool from distutils import log -from distutils.ccompiler import new_compiler +from distutils.ccompiler import new_compiler, CCompiler from distutils.sysconfig import customize_compiler from distutils.errors import CompileError, LinkError +from distutils.version import LooseVersion from pkg_resources import resource_filename from subprocess import Popen, PIPE +from setuptools.command.build_ext import build_ext as _build_ext +from setuptools.command.sdist import sdist as _sdist CCODE = """ @@ -124,12 +130,32 @@ def check_for_openmp(): return using_openmp -def check_for_pyembree(): +def check_for_pyembree(std_libs): + embree_libs = [] + embree_aliases = {} try: fn = resource_filename("pyembree", "rtcore.pxd") except ImportError: - return None - return os.path.dirname(fn) + return embree_libs, embree_aliases + + embree_prefix = os.path.abspath(read_embree_location()) + embree_inc_dir = os.path.join(embree_prefix, 'include') + embree_lib_dir = os.path.join(embree_prefix, 'lib') + if in_conda_env(): + conda_basedir = os.path.dirname(os.path.dirname(sys.executable)) + embree_inc_dir.append(os.path.join(conda_basedir, 'include')) + embree_lib_dir.append(os.path.join(conda_basedir, 'lib')) + + if _platform == "darwin": + embree_lib_name = "embree.2" + else: + embree_lib_name = "embree" + + embree_aliases['EMBREE_INC_DIR'] = ['yt/utilities/lib/', embree_inc_dir] + embree_aliases['EMBREE_LIB_DIR'] = [embree_lib_dir] + embree_aliases['EMBREE_LIBS'] = std_libs + [embree_lib_name] + embree_libs += ['yt/utilities/lib/embree_mesh/*.pyx'] + return embree_libs, embree_aliases def in_conda_env(): return any(s in sys.version for s in ("Anaconda", "Continuum")) @@ -206,3 +232,104 @@ def read_embree_location(): shutil.rmtree(tmpdir) return rd + +def get_cpu_count(): + if platform.system() != "Windows": + return os.cpu_count() + return 0 + +def install_ccompiler(): + def _compile( + self, sources, output_dir=None, macros=None, include_dirs=None, + debug=0, extra_preargs=None, extra_postargs=None, depends=None, + ): + """Function to monkey-patch distutils.ccompiler.CCompiler""" + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs + ) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # Return *all* object filenames, not just the ones we just built. + return objects + + CCompiler.compile = _compile + +def create_build_ext(lib_exts, cythonize_aliases): + class build_ext(_build_ext): + # subclass setuptools extension builder to avoid importing cython and numpy + # at top level in setup.py. See http://stackoverflow.com/a/21621689/1382869 + def finalize_options(self): + try: + import cython + import numpy + except ImportError: + raise ImportError( + """Could not import cython or numpy. Building yt from source requires + cython and numpy to be installed. Please install these packages using + the appropriate package manager for your python environment.""") + if LooseVersion(cython.__version__) < LooseVersion('0.26.1'): + raise RuntimeError( + """Building yt from source requires Cython 0.26.1 or newer but + Cython %s is installed. Please update Cython using the appropriate + package manager for your python environment.""" % + cython.__version__) + if LooseVersion(numpy.__version__) < LooseVersion('1.13.3'): + raise RuntimeError( + """Building yt from source requires NumPy 1.13.3 or newer but + NumPy %s is installed. Please update NumPy using the appropriate + package manager for your python environment.""" % + numpy.__version__) + from Cython.Build import cythonize + # Override the list of extension modules + self.distribution.ext_modules[:] = cythonize( + lib_exts, + aliases = cythonize_aliases, + compiler_directives={'language_level': 2}, + nthreads=get_cpu_count(), + ) + _build_ext.finalize_options(self) + # Prevent numpy from thinking it is still in its setup process + # see http://stackoverflow.com/a/21621493/1382869 + if isinstance(__builtins__, dict): + # sometimes this is a dict so we need to check for that + # https://docs.python.org/3/library/builtins.html + __builtins__["__NUMPY_SETUP__"] = False + else: + __builtins__.__NUMPY_SETUP__ = False + import numpy + self.include_dirs.append(numpy.get_include()) + + def build_extensions(self): + self.check_extensions_list(self.extensions) + + ncpus = get_cpu_count() + if ncpus > 0: + with Pool(ncpus) as pool: + pool.map(self.build_extension, self.extensions) + else: + super().build_extensions() + + + class sdist(_sdist): + # subclass setuptools source distribution builder to ensure cython + # generated C files are included in source distribution. + # See http://stackoverflow.com/a/18418524/1382869 + def run(self): + # Make sure the compiled Cython files in the distribution are up-to-date + from Cython.Build import cythonize + cythonize( + lib_exts, + aliases = cythonize_aliases, + compiler_directives={'language_level': 2}, + nthreads=get_cpu_count(), + ) + _sdist.run(self) + + return build_ext, sdist From daa203de942d2522dc94c4db8dd4c4baa25c61f3 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Fri, 17 Apr 2020 11:29:33 -0500 Subject: [PATCH 13/17] Fix vec3_ops cimport for case when we don't have pyembree --- yt/utilities/lib/embree_mesh/mesh_intersection.pyx | 3 ++- yt/utilities/lib/{embree_mesh => }/vec3_ops.pxd | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename yt/utilities/lib/{embree_mesh => }/vec3_ops.pxd (100%) diff --git a/yt/utilities/lib/embree_mesh/mesh_intersection.pyx b/yt/utilities/lib/embree_mesh/mesh_intersection.pyx index 33a4075d521..06e90078daa 100644 --- a/yt/utilities/lib/embree_mesh/mesh_intersection.pyx +++ b/yt/utilities/lib/embree_mesh/mesh_intersection.pyx @@ -30,7 +30,8 @@ from yt.utilities.lib.primitives cimport \ tet_patchSurfaceDerivU, \ tet_patchSurfaceDerivV, \ compute_tet_patch_hit -from vec3_ops cimport dot, subtract, cross, distance +from yt.utilities.lib.vec3_ops cimport \ + dot, subtract, cross, distance @cython.boundscheck(False) diff --git a/yt/utilities/lib/embree_mesh/vec3_ops.pxd b/yt/utilities/lib/vec3_ops.pxd similarity index 100% rename from yt/utilities/lib/embree_mesh/vec3_ops.pxd rename to yt/utilities/lib/vec3_ops.pxd From 7afe579282de1a054c880559176c1bd8e279b1dc Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Wed, 29 Apr 2020 12:59:47 -0500 Subject: [PATCH 14/17] change omp_args to empty list --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3cf40f2d1be..54c316a1bd6 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ if check_for_openmp() is True: omp_args = ['-fopenmp'] else: - omp_args = None + omp_args = [] if os.name == "nt": std_libs = [] From 68170566e86eca1c413ec2e83aa8aefa96419587 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Mon, 22 Jun 2020 10:59:46 -0500 Subject: [PATCH 15/17] fix renamed module cimports --- yt/utilities/lib/embree_mesh/mesh_intersection.pxd | 2 +- yt/utilities/lib/embree_mesh/mesh_intersection.pyx | 2 +- yt/utilities/lib/embree_mesh/mesh_samplers.pyx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yt/utilities/lib/embree_mesh/mesh_intersection.pxd b/yt/utilities/lib/embree_mesh/mesh_intersection.pxd index 974f675be7f..f5532682c3c 100644 --- a/yt/utilities/lib/embree_mesh/mesh_intersection.pxd +++ b/yt/utilities/lib/embree_mesh/mesh_intersection.pxd @@ -1,7 +1,7 @@ cimport pyembree.rtcore as rtc cimport pyembree.rtcore_ray as rtcr cimport pyembree.rtcore_geometry as rtcg -from yt.utilities.lib.mesh_construction cimport Patch, Tet_Patch +from .mesh_construction cimport Patch, Tet_Patch cimport cython cdef void patchIntersectFunc(Patch* patches, diff --git a/yt/utilities/lib/embree_mesh/mesh_intersection.pyx b/yt/utilities/lib/embree_mesh/mesh_intersection.pyx index 06e90078daa..a130f0beaef 100644 --- a/yt/utilities/lib/embree_mesh/mesh_intersection.pyx +++ b/yt/utilities/lib/embree_mesh/mesh_intersection.pyx @@ -18,7 +18,7 @@ from pyembree.rtcore cimport Vec3f cimport numpy as np cimport cython from libc.math cimport fabs, fmin, fmax, sqrt -from yt.utilities.lib.mesh_samplers cimport sample_hex20, sample_tet10 +from .mesh_samplers cimport sample_hex20, sample_tet10 from yt.utilities.lib.bounding_volume_hierarchy cimport BBox from yt.utilities.lib.primitives cimport \ patchSurfaceFunc, \ diff --git a/yt/utilities/lib/embree_mesh/mesh_samplers.pyx b/yt/utilities/lib/embree_mesh/mesh_samplers.pyx index 53557d85aa5..34ae8feff65 100644 --- a/yt/utilities/lib/embree_mesh/mesh_samplers.pyx +++ b/yt/utilities/lib/embree_mesh/mesh_samplers.pyx @@ -14,7 +14,7 @@ Note - this file is only used for the Embree-accelerated ray-tracer. cimport pyembree.rtcore as rtc cimport pyembree.rtcore_ray as rtcr from pyembree.rtcore cimport Vec3f, Triangle, Vertex -from yt.utilities.lib.mesh_construction cimport \ +from .mesh_construction cimport \ MeshDataContainer, \ Patch, \ Tet_Patch From 9c721a0124626e882920a3ec08546bc18a341863 Mon Sep 17 00:00:00 2001 From: Madicken Munk Date: Wed, 1 Jul 2020 13:51:56 -0500 Subject: [PATCH 16/17] add embree_mesh files to gitignore --- .gitignore | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 478794534b3..a93b728430d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,10 +55,10 @@ yt/utilities/lib/interpolators.c yt/utilities/lib/kdtree.c yt/utilities/lib/lenses.c yt/utilities/lib/line_integral_convolution.c -yt/utilities/lib/mesh_construction.cpp -yt/utilities/lib/mesh_intersection.cpp -yt/utilities/lib/mesh_samplers.cpp -yt/utilities/lib/mesh_traversal.cpp +yt/utilities/lib/embree_mesh/mesh_construction.cpp +yt/utilities/lib/embree_mesh/mesh_intersection.cpp +yt/utilities/lib/embree_mesh/mesh_samplers.cpp +yt/utilities/lib/embree_mesh/mesh_traversal.cpp yt/utilities/lib/mesh_triangulation.c yt/utilities/lib/mesh_utilities.c yt/utilities/lib/pixelization_routines.cpp From 3551424d55b168c6392d30adb20d92e6b3635b2a Mon Sep 17 00:00:00 2001 From: Madicken Munk Date: Wed, 1 Jul 2020 13:53:30 -0500 Subject: [PATCH 17/17] modify mesh_traversal path in test_install_script --- tests/test_install_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_install_script.py b/tests/test_install_script.py index a1cb4cc77c3..21f9fe183e3 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -36,7 +36,7 @@ ] DEPENDENCY_IMPORT_TESTS = { - 'embree': "from yt.utilities.lib import mesh_traversal", + 'embree': "from yt.utilities.lib.embree_mesh import mesh_traversal", }