merging yt-4 into master

.gitignore

@@ -21,6 +21,7 @@ yt/geometry/oct_container.c
 yt/geometry/oct_visitors.c
 yt/geometry/particle_deposit.c
 yt/geometry/particle_oct_container.c
+yt/geometry/particle_oct_container.cpp
 yt/geometry/particle_smooth.c
 yt/geometry/selection_routines.c
 yt/utilities/amr_utils.c
@@ -34,11 +35,16 @@ yt/utilities/lib/alt_ray_tracers.c
 yt/utilities/lib/amr_kdtools.c
 yt/utilities/lib/basic_octree.c
 yt/utilities/lib/bitarray.c
+yt/utilities/lib/bounded_priority_queue.c
 yt/utilities/lib/bounding_volume_hierarchy.c
 yt/utilities/lib/contour_finding.c
+yt/utilities/lib/cykdtree/kdtree.cpp
+yt/utilities/lib/cykdtree/utils.cpp
+yt/utilities/lib/cyoctree.cpp
 yt/utilities/lib/depth_first_octree.c
 yt/utilities/lib/distance_queue.c
 yt/utilities/lib/element_mappings.c
+yt/utilities/lib/ewah_bool_wrap.cpp
 yt/utilities/lib/fnv_hash.c
 yt/utilities/lib/fortran_reader.c
 yt/utilities/lib/freetype_writer.c
@@ -55,13 +61,16 @@ yt/utilities/lib/mesh_samplers.cpp
 yt/utilities/lib/mesh_traversal.cpp
 yt/utilities/lib/mesh_triangulation.c
 yt/utilities/lib/mesh_utilities.c
+yt/utilities/lib/pixelization_routines.cpp
 yt/utilities/lib/misc_utilities.c
+yt/utilities/lib/particle_kdtree_tools.cpp
 yt/utilities/lib/particle_mesh_operations.c
 yt/utilities/lib/partitioned_grid.c
 yt/utilities/lib/primitives.c
 yt/utilities/lib/origami.c
 yt/utilities/lib/particle_mesh_operations.c
 yt/utilities/lib/pixelization_routines.c
+yt/utilities/lib/pixelization_routines.cpp
 yt/utilities/lib/png_writer.c
 yt/utilities/lib/points_in_volume.c
 yt/utilities/lib/quad_tree.c

.hgignore

@@ -37,6 +37,7 @@ yt/utilities/lib/contour_finding.c
 yt/utilities/lib/depth_first_octree.c
 yt/utilities/lib/distance_queue.c
 yt/utilities/lib/element_mappings.c
+yt/utilities/lib/ewah_bool_wrap.cpp
 yt/utilities/lib/fnv_hash.c
 yt/utilities/lib/fortran_reader.c
 yt/utilities/lib/freetype_writer.c
@@ -54,6 +55,7 @@ yt/utilities/lib/mesh_traversal.cpp
 yt/utilities/lib/mesh_triangulation.c
 yt/utilities/lib/mesh_utilities.c
 yt/utilities/lib/misc_utilities.c
+yt/utilities/lib/particle_kdtree_tools.cpp
 yt/utilities/lib/particle_mesh_operations.c
 yt/utilities/lib/partitioned_grid.c
 yt/utilities/lib/primitives.c

.travis.yml

@@ -1,6 +1,7 @@
 language: python
 dist: xenial
 sudo: required
+dist: xenial
 cache:
   pip: true
   directories:
@@ -10,6 +11,7 @@ addons:
   apt:
     packages:
       - libhdf5-serial-dev
+      - libnetcdf-dev
       - libproj-dev
       - proj-data
       - proj-bin
@@ -20,10 +22,6 @@ env:
   global:
     PIP=pip
 
-env:
-  global:
-    PIP=pip
-
 before_install:
   - |
     if [[ $TRAVIS_OS_NAME != 'osx' ]]; then
@@ -56,7 +54,9 @@ install:
     # install dependencies yt
     if [[ $TRAVIS_BUILD_STAGE_NAME != "Lint" ]]; then
       if [[ $MINIMAL == 1 ]]; then
-        $PIP install numpy==1.10.4 cython==0.24
+        # Ensure numpy and cython are installed so dependencies that need to be built
+        # don't error out
+        $PIP install numpy==1.13.3 cython==0.26.1
         $PIP install -r tests/test_minimal_requirements.txt
       else
         # Getting cartopy installed requires getting cython and numpy installed
@@ -75,24 +75,15 @@ install:
 jobs:
   include:
     - stage: lint
-      python: 3.6
-      script: flake8 yt/
-
-    - stage: lint
-      python: 2.7
+      python: 3.7
       script: flake8 yt/
 
     - stage: tests
-      name: "Python: 2.7 Unit Tests"
-      python: 2.7
+      name: "Minimal Dependency Version Unit Tests"
+      python: 3.5
       env: MINIMAL=1
       script: coverage run $(which nosetests) -c nose_unit.cfg
 
-    - stage: tests
-      name: "Python: 2.7 Unit Tests"
-      python: 2.7
-      script: coverage run $(which nosetests) -c nose_unit.cfg
-
     - stage: tests
       name: "Python: 3.5 Unit Tests"
       python: 3.5
@@ -104,15 +95,19 @@ jobs:
       script: coverage run $(which nosetests) -c nose_unit.cfg
 
     - stage: tests
-      name: "Python: 2.7 Minimal Dependency Answer Tests"
-      python: 2.7
+      name: "Python: 3.7 Unit Tests"
+      python: 3.7
+      script: coverage run $(which nosetests) -c nose_unit.cfg
+
+    - name: "Minimal Dependency Version Answer Tests"
+      python: 3.5
       env: MINIMAL=1
       script: coverage run $(which nosetests) -c nose_answer.cfg
       after_failure: python tests/report_failed_answers.py -f -m --xunit-file "answer_nosetests.xml"
 
     - stage: tests
-      name: "Python: 3.6 Answer Tests"
-      python: 3.6
+      name: "Python: 3.7 Answer Tests"
+      python: 3.7
       script: coverage run $(which nosetests) -c nose_answer.cfg
       after_failure: python tests/report_failed_answers.py -f -m --xunit-file "answer_nosetests.xml"
 

CONTRIBUTING.rst

@@ -405,12 +405,6 @@ the following subdirectories:
    classes for data regions, covering grids, time series, and so on.  This
    also includes derived fields and derived quantities.
 
-``analysis_modules``
-   This is where all mechanisms for processing data live.  This includes
-   things like clump finding, halo profiling, halo finding, and so on.  This
-   is something of a catchall, but it serves as a level of greater
-   abstraction that simply data selection and modification.
-
 ``gui``
    This is where all GUI components go.  Typically this will be some small
    tool used for one or two things, which contains a launching mechanism on
@@ -762,6 +756,7 @@ Source code style guide
 
  * In general, follow PEP-8 guidelines.
    https://www.python.org/dev/peps/pep-0008/
+ * We no longer have a copyright blurb in every source file.
  * Classes are ``ConjoinedCapitals``, methods and functions are
    ``lowercase_with_underscores``.
  * Use 4 spaces, not tabs, to represent indentation.
@@ -784,7 +779,7 @@ Source code style guide
    that occur on an object.  See :ref:`docstrings` below for a fiducial example
    of a docstring.
  * Use only one top-level import per line. Unless there is a good reason not to,
-   imports should happen at the top of the file, after the copyright blurb.
+   imports should happen at the top of the file.
  * Never compare with ``True`` or ``False`` using ``==`` or ``!=``, always use
    ``is`` or ``is not``.
  * If you are comparing with a numpy boolean array, just refer to the array.

MANIFEST.in

@@ -3,14 +3,13 @@ include yt/visualization/mapserver/html/map.js
 include yt/visualization/mapserver/html/map_index.html
 include yt/utilities/mesh_types.yaml
 exclude scripts/pr_backport.py
-recursive-include yt *.py *.pyx *.pxd *.h README* *.txt LICENSE* *.cu
+recursive-include yt *.py *.pyx *.pxd *.h *.hpp README* *.txt LICENSE* *.cu
 recursive-include doc *.rst *.txt *.py *.ipynb *.png *.jpg *.css *.html
 recursive-include doc *.h *.c *.sh *.svgz *.pdf *.svg *.pyx
 include doc/README doc/activate doc/activate.csh doc/cheatsheet.tex
 include doc/extensions/README doc/Makefile
 prune doc/source/reference/api/generated
 prune doc/build
-recursive-include yt/analysis_modules/halo_finding/rockstar *.py *.pyx
 recursive-include yt/visualization/volume_rendering/shaders *.fragmentshader *.vertexshader
 prune yt/frontends/_skeleton
 recursive-include yt/frontends/amrvac *.par

appveyor.yml

@@ -7,8 +7,8 @@ environment:
       PYTHON: "C:\\Miniconda36-x64"
 
   matrix:
-      - PYTHON_VERSION: "2.7"
       - PYTHON_VERSION: "3.6"
+        NUMPY_VERSION: "1.16"
 
 platform:
     -x64
@@ -30,7 +30,9 @@ install:
     - "python --version"
 
     # Install specified version of numpy and dependencies
-    - "conda install --yes -c conda-forge numpy scipy nose setuptools ipython Cython sympy=1.4 fastcache h5py matplotlib mock pandas cartopy conda-build pyyaml"
+    - "conda install --yes -c conda-forge numpy==%NUMPY_VERSION% scipy nose setuptools ipython
+    Cython sympy=1.4 fastcache h5py matplotlib mock pandas cartopy conda-build unyt pyyaml"
+    # install yt
     - "conda develop -b ."
 
 # Not a .NET project

doc/Makefile

@@ -49,8 +49,7 @@ html:
 ifneq ($(READTHEDOCS),True)
 	SPHINX_APIDOC_OPTIONS=members,undoc-members,inherited-members,show-inheritance sphinx-apidoc \
         -o source/reference/api/ \
-        -e ../yt ../yt/extern* $(shell find ../yt -name "*tests*" -type d) ../yt/utilities/voropp* \
-           ../yt/analysis_modules/halo_finding/{fof,hop}
+        -e ../yt ../yt/extern* $(shell find ../yt -name "*tests*" -type d) ../yt/utilities/voropp*
 endif
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo

doc/helper_scripts/show_fields.py

@@ -54,8 +54,10 @@ def _strip_ftype(field):
                          unit_registry=ds.unit_registry)
 for my_unit in ["m", "pc", "AU", "au"]:
     new_unit = "%scm" % my_unit
-    ds.unit_registry.add(new_unit, base_ds.unit_registry.lut[my_unit][0],
-                         dimensions.length, "\\rm{%s}/(1+z)" % my_unit)
+    my_u = Unit(my_unit, registry=ds.unit_registry)
+    ds.unit_registry.add(new_unit, my_u.base_value,
+                         dimensions.length, "\\rm{%s}/(1+z)" % my_unit,
+                         prefixable=True)
 
 
 
@@ -143,7 +145,7 @@ def print_all_fields(fl):
                 print("   * Units: :math:`%s`" % fix_units(df.units))
             else:
                 print("   * Units: :math:`%s`" % fix_units(df.units, in_cgs=True))
-        print("   * Particle Type: %s" % (df.particle_type))
+        print("   * Sampling Method: %s" % (df.sampling_type))
         print()
         print("**Field Source**")
         print()

doc/helper_scripts/table.py

@@ -23,13 +23,6 @@
      ("interacting/index.html", "Interacting with yt",
       "Different ways -- scripting, GUIs, prompts, explorers -- to explore " +
       "your data."),
-     ("analysis_modules/index.html", "Analysis Modules",
-      "Discussions of some provided procedures for astrophysical analysis " +
-      "like halo finding and synthetic spectra.  Halo finding, analyzing " +
-      "cosmology simulations, halo mass functions, halo profiling, light " +
-      "cone generator, making absorption spectrums, star particle " +
-      "analysis, two-point functions, halo merger trees, clump finding, " +
-      "radial column density, exporting to sunrise.")
   ]),
   ("Advanced Usage", [
      ("advanced/index.html", "Advanced yt usage",

doc/install_script.sh

@@ -32,7 +32,6 @@ INST_GIT=1      # Install git or not?  If git is not already installed, yt
 INST_EMBREE=0   # Install dependencies needed for Embree-accelerated ray tracing
 INST_PYX=0      # Install PyX?  Sometimes PyX can be problematic without a
                 # working TeX installation.
-INST_ROCKSTAR=0 # Install the Rockstar halo finder?
 INST_SCIPY=0    # Install scipy?
 INST_H5PY=1     # Install h5py?
 INST_ASTROPY=0  # Install astropy?
@@ -134,7 +133,6 @@ function write_config
     echo INST_GIT=${INST_GIT} >> ${CONFIG_FILE}
     echo INST_PYX=${INST_PYX} >> ${CONFIG_FILE}
     echo INST_PY3=${INST_PY3} >> ${CONFIG_FILE}
-    echo INST_ROCKSTAR=${INST_ROCKSTAR} >> ${CONFIG_FILE}
     echo INST_SCIPY=${INST_SCIPY} >> ${CONFIG_FILE}
     echo INST_EMBREE=${INST_EMBREE} >> ${CONFIG_FILE}
     echo INST_H5PY=${INST_H5PY} >> ${CONFIG_FILE}
@@ -286,17 +284,6 @@ then
     PYEMBREE_URL="https://github.com/scopatz/pyembree/archive/master.zip"
 fi
 
-if [ $INST_ROCKSTAR -ne 0 ]
-then
-    if [ $INST_YT_SOURCE -eq 0 ]
-    then
-        echo "yt must be compiled from source to install support for"
-        echo "the rockstar halo finder. Please set INST_YT_SOURCE to 1"
-        echo "and re-run the install script"
-        exit 1
-    fi
-fi
-
 echo
 echo
 echo "========================================================================"
@@ -330,10 +317,6 @@ printf "%-18s = %s so I " "INST_PYX" "${INST_PYX}"
 get_willwont ${INST_PYX}
 echo "be installing PyX"
 
-printf "%-18s = %s so I " "INST_ROCKSTAR" "${INST_ROCKSTAR}"
-get_willwont ${INST_ROCKSTAR}
-echo "be installing Rockstar"
-
 printf "%-18s = %s so I " "INST_H5PY" "${INST_H5PY}"
 get_willwont ${INST_H5PY}
 echo "be installing h5py"
@@ -598,16 +581,6 @@ then
     popd &> /dev/null
 fi
 
-if [ $INST_ROCKSTAR -eq 1 ]
-then
-    echo "Building Rockstar"
-    ( ${GIT_EXE} clone https://github.com/yt-project/rockstar ${DEST_DIR}/src/rockstar/ 2>&1 ) 1>> ${LOG_FILE}
-    ROCKSTAR_PACKAGE=$(${DEST_DIR}/bin/conda build ${DEST_DIR}/src/yt_conda/rockstar --output)
-    log_cmd ${DEST_DIR}/bin/conda build ${DEST_DIR}/src/yt_conda/rockstar
-    log_cmd ${DEST_DIR}/bin/conda install $ROCKSTAR_PACKAGE
-    ROCKSTAR_DIR=${DEST_DIR}/src/rockstar
-fi
-
 # conda doesn't package pyx, so we install manually with pip
 if [ $INST_PYX -eq 1 ]
 then
@@ -650,13 +623,8 @@ else
     then
         echo $DEST_DIR > ${YT_DIR}/embree.cfg
     fi
-    if [ $INST_ROCKSTAR -eq 1 ]
-    then
-        echo $ROCKSTAR_DIR > ${YT_DIR}/rockstar.cfg
-        ROCKSTAR_LIBRARY_PATH=${DEST_DIR}/lib
-    fi
     pushd ${YT_DIR} &> /dev/null
-    ( LIBRARY_PATH=$ROCKSTAR_LIBRARY_PATH ${DEST_DIR}/bin/${PYTHON_EXEC} setup.py develop 2>&1) 1>> ${LOG_FILE} || do_exit
+    ( ${DEST_DIR}/bin/${PYTHON_EXEC} setup.py develop 2>&1) 1>> ${LOG_FILE} || do_exit
     popd &> /dev/null
 fi
 
@@ -697,21 +665,6 @@ echo "You can also update the init file appropriate for your shell"
 echo "(e.g. .bashrc, .bash_profile, .cshrc, or .zshrc) to include"
 echo "the same command."
 echo
-if [ $INST_ROCKSTAR -eq 1 ]
-then
-    if [ $MYOS = "Darwin" ]
-    then
-        LD_NAME="DYLD_LIBRARY_PATH"
-    else
-        LD_NAME="LD_LIBRARY_PATH"
-    fi
-    echo
-    echo "For rockstar to work, you must also set $LD_NAME:"
-    echo
-    echo "    export $LD_NAME=$DEST_DIR/lib:\$$LD_NAME"
-    echo
-    echo "or whichever invocation is appropriate for your shell."
-fi
 echo "========================================================================"
 echo
 echo "Oh, look at me, still talking when there's science to do!"