Merge pull request #1461 from astrofrog/fast-histogram

Implement support for scatter density plots

.travis.yml

@@ -20,14 +20,14 @@ env:
     - PYTHON_VERSION=3.6
   global:
     # We add astropy-ci-extras to have the latest version of Astropy with older Numpy versions.
-    - CONDA_CHANNELS="astropy-ci-extras astropy"
+    - CONDA_CHANNELS="astropy-ci-extras astropy glueviz"
     - PYTEST_ARGS="--cov glue -vs"
     - ASTROPY_VERSION=stable
     - NUMPY_VERSION=stable
     - NO_CFG_FILES=false
     - QT_PKG=pyqt5
     - SETUP_XVFB=True
-    - CONDA_DEPENDENCIES="pip dill ipython matplotlib scipy cython h5py pygments pyzmq scikit-image pandas sphinx xlrd pillow pytest mock coverage pyyaml sphinx_rtd_theme qtpy traitlets ipykernel qtconsole spectral-cube pytest-cov"
+    - CONDA_DEPENDENCIES="pip dill ipython matplotlib scipy cython h5py pygments pyzmq scikit-image pandas sphinx xlrd pillow pytest mock coverage pyyaml sphinx_rtd_theme qtpy traitlets ipykernel qtconsole spectral-cube pytest-cov mpl-scatter-density"
     - PIP_DEPENDENCIES="coveralls pyavm astrodendro awscli plotly"
     - PIP_FALLBACK=false
     - REMOVE_INSTALL_REQUIRES=0
@@ -58,7 +58,7 @@ matrix:
         - os: linux
           env: PYTHON_VERSION=3.6
                PYTEST_ARGS="--cov glue"
-               CONDA_DEPENDENCIES="pip setuptools pandas mock matplotlib qtpy ipython ipykernel qtconsole"
+               CONDA_DEPENDENCIES="pip setuptools pandas mock matplotlib qtpy ipython ipykernel qtconsole mpl-scatter-density"
                PIP_DEPENDENCIES="pytest-cov coveralls"
                REMOVE_INSTALL_REQUIRES=1
 
@@ -70,19 +70,13 @@ matrix:
 
         # Test with older package versions:
 
-        - os: linux
-          env: PYTHON_VERSION=2.7
-               MATPLOTLIB_VERSION=1.4
-               NUMPY_VERSION=1.9
-               IPYTHON_VERSION=4
-               PANDAS_VERSION=0.14
-               SETUPTOOLS=1.0
-               QT_PKG=pyqt
-
         - os: linux
           env: PYTHON_VERSION=2.7
                MATPLOTLIB_VERSION=1.5
-               NUMPY_VERSION=1.10
+               NUMPY_VERSION=1.11
+               PANDAS_VERSION=0.18
+               SETUPTOOLS=1.0
+               IPYTHON_VERSION=4
                QT_PKG=pyqt
 
         # Test with PySide, but due to segmentation faults, mark as an

CHANGES.md

@@ -4,7 +4,8 @@ Full changelog
 v0.13.0 (unreleased)
 --------------------
 
-* No changes yet
+* Added support for scatter density maps, which is useful when making
+  scatter plots of many points. [#1461]
 
 v0.12.1 (unreleased)
 --------------------

appveyor.yml

@@ -10,7 +10,8 @@ environment:
       PYTHON_ARCH: "64" # needs to be set for CMD_IN_ENV to succeed. If a mix
                         # of 32 bit and 64 bit builds are needed, move this
                         # to the matrix section.
-      CONDA_DEPENDENCIES: "astropy scipy cython pyqt matplotlib h5py pygments pyzmq scikit-image pandas xlrd pillow pytest mock coverage ipython ipykernel qtconsole traitlets qtpy"
+      CONDA_CHANNELS: "glueviz"
+      CONDA_DEPENDENCIES: "astropy scipy cython pyqt matplotlib h5py pygments pyzmq scikit-image pandas xlrd pillow pytest mock coverage ipython ipykernel qtconsole traitlets qtpy mpl-scatter-density"
       PIP_DEPENDENCIES: "plotly"
       PIP_FALLBACK: "False"
 

circle.yml

@@ -10,7 +10,8 @@ dependencies:
     # upgrade setuptools.
     - sudo easy_install --version
     - sudo pip3 --version
-    - sudo pip3 install -q .[all]
+    - sudo pip3 install numpy
+    - sudo pip3 install .[all]
 
 test:
   override:

doc/installation/dependencies.rst

@@ -21,7 +21,8 @@ Glue has the following required dependencies:
 * `dill <https://pypi.python.org/pypi/dill>`_ 0.2 or later (which improves session saving)
 * `h5py <http://www.h5py.org>`_ 2.4 or later, for reading HDF5 files
 * `xlrd <https://pypi.python.org/pypi/xlrd>`_ 1.0 or later, for reading Excel files
-* `glue-vispy-viewers <https://pypi.python.org/pypi/glue-vispy-viewers>`_, which provide 3D viewers
+* `mpl-scatter-density <https://github.com/astrofrog/mpl-scatter-density>`_, for making
+  scatter density maps of many points.
 
 The following optional dependencies are also highly recommended and
 domain-independent:

glue/_deps.py

@@ -155,7 +155,7 @@ def version(self):
 )
 
 required = (
-    Dependency('qtpy', 'Required', min_version='1.1'),
+    Dependency('qtpy', 'Required', min_version='1.2'),
     Dependency('setuptools', 'Required', min_version='1.0'),
     Dependency('numpy', 'Required', min_version='1.9'),
     Dependency('matplotlib', 'Required for plotting', min_version='1.4'),
@@ -164,6 +164,7 @@ def version(self):
     Dependency('dill', 'Used when saving Glue sessions', min_version='0.2'),
     Dependency('h5py', 'Used to support HDF5 files', min_version='2.4'),
     Dependency('xlrd', 'Used to support Excel files', min_version='1.0'),
+    Dependency('mpl_scatter_density', 'Used to make fast scatter density plots', 'mpl-scatter-density', min_version='0.3'),
 )
 
 general = (

glue/viewers/image/state.py

@@ -49,6 +49,8 @@ class ImageViewerState(MatplotlibDataViewerState):
                                                     'whether each layer is assigned '
                                                     'a single color (``One color per layer``)')
 
+    dpi = DDCProperty(72, docstring='The resolution (in dots per inch) of density maps, if present')
+
     def __init__(self, **kwargs):
 
         super(ImageViewerState, self).__init__()

glue/viewers/scatter/layer_artist.py

@@ -5,18 +5,29 @@
 from matplotlib.colors import Normalize
 from matplotlib.collections import LineCollection
 
+from mpl_scatter_density import ScatterDensityArtist
+
+from astropy.visualization import (ImageNormalize, LinearStretch, SqrtStretch,
+                                   AsinhStretch, LogStretch)
+
 from glue.utils import defer_draw, broadcast_to
 from glue.viewers.scatter.state import ScatterLayerState
 from glue.viewers.matplotlib.layer_artist import MatplotlibLayerArtist
 from glue.core.exceptions import IncompatibleAttribute
 
+STRETCHES = {'linear': LinearStretch,
+             'sqrt': SqrtStretch,
+             'arcsinh': AsinhStretch,
+             'log': LogStretch}
+
 CMAP_PROPERTIES = set(['cmap_mode', 'cmap_att', 'cmap_vmin', 'cmap_vmax', 'cmap'])
 MARKER_PROPERTIES = set(['size_mode', 'size_att', 'size_vmin', 'size_vmax', 'size_scaling', 'size'])
 LINE_PROPERTIES = set(['linewidth', 'linestyle'])
-VISUAL_PROPERTIES = (CMAP_PROPERTIES | MARKER_PROPERTIES |
+DENSITY_PROPERTIES = set(['dpi', 'stretch', 'density_contrast'])
+VISUAL_PROPERTIES = (CMAP_PROPERTIES | MARKER_PROPERTIES | DENSITY_PROPERTIES |
                      LINE_PROPERTIES | set(['color', 'alpha', 'zorder', 'visible']))
 
-DATA_PROPERTIES = set(['layer', 'x_att', 'y_att', 'cmap_mode', 'size_mode',
+DATA_PROPERTIES = set(['layer', 'x_att', 'y_att', 'cmap_mode', 'size_mode', 'density_map',
                        'xerr_att', 'yerr_att', 'xerr_visible', 'yerr_visible',
                        'vector_visible', 'vx_att', 'vy_att', 'vector_arrowhead', 'vector_mode',
                        'vector_origin', 'line_visible', 'markers_visible', 'vector_scaling'])
@@ -27,11 +38,25 @@ def __call__(self, *args, **kwargs):
         return 1 - super(InvertedNormalize, self).__call__(*args, **kwargs)
 
 
+class DensityMapLimits(object):
+
+    contrast = 1
+
+    def min(self, array):
+        return 0
+
+    def max(self, array):
+        return 10. ** (np.log10(np.nanmax(array)) * self.contrast)
+
+
 def set_mpl_artist_cmap(artist, values, state):
     vmin = state.cmap_vmin
     vmax = state.cmap_vmax
     cmap = state.cmap
-    artist.set_array(values)
+    if isinstance(artist, ScatterDensityArtist):
+        artist.set_c(values)
+    else:
+        artist.set_array(values)
     artist.set_cmap(cmap)
     if vmin > vmax:
         artist.set_clim(vmax, vmin)
@@ -63,9 +88,16 @@ def __init__(self, axes, viewer_state, layer_state=None, layer=None):
         self.line_collection = LineCollection(np.zeros((0, 2, 2)))
         self.axes.add_collection(self.line_collection)
 
+        # Scatter density
+        self.density_auto_limits = DensityMapLimits()
+        self.density_artist = ScatterDensityArtist(self.axes, [], [], color='white',
+                                                   vmin=self.density_auto_limits.min,
+                                                   vmax=self.density_auto_limits.max)
+        self.axes.add_artist(self.density_artist)
+
         self.mpl_artists = [self.scatter_artist, self.plot_artist,
                             self.errorbar_artist, self.vector_artist,
-                            self.line_collection]
+                            self.line_collection, self.density_artist]
         self.errorbar_index = 2
         self.vector_index = 3
 
@@ -101,20 +133,26 @@ def _update_data(self, changed):
             self.enable()
 
         if self.state.markers_visible:
-            if self.state.cmap_mode == 'Fixed' and self.state.size_mode == 'Fixed':
-                # In this case we use Matplotlib's plot function because it has much
-                # better performance than scatter.
-                self.plot_artist.set_data(x, y)
-                offsets = np.zeros((0, 2))
-                self.scatter_artist.set_offsets(offsets)
-            else:
+            if self.state.density_map:
+                self.density_artist.set_xy(x, y)
                 self.plot_artist.set_data([], [])
-                offsets = np.vstack((x, y)).transpose()
-                self.scatter_artist.set_offsets(offsets)
+                self.scatter_artist.set_offsets(np.zeros((0, 2)))
+            else:
+                if self.state.cmap_mode == 'Fixed' and self.state.size_mode == 'Fixed':
+                    # In this case we use Matplotlib's plot function because it has much
+                    # better performance than scatter.
+                    self.plot_artist.set_data(x, y)
+                    self.scatter_artist.set_offsets(np.zeros((0, 2)))
+                    self.density_artist.set_xy([], [])
+                else:
+                    self.plot_artist.set_data([], [])
+                    offsets = np.vstack((x, y)).transpose()
+                    self.scatter_artist.set_offsets(offsets)
+                    self.density_artist.set_xy([], [])
         else:
             self.plot_artist.set_data([], [])
-            offsets = np.zeros((0, 2))
-            self.scatter_artist.set_offsets(offsets)
+            self.scatter_artist.set_offsets(np.zeros((0, 2)))
+            self.density_artist.set_xy([], [])
 
         if self.state.line_visible:
             if self.state.cmap_mode == 'Fixed':
@@ -214,45 +252,69 @@ def _update_visual_attributes(self, changed, force=False):
 
         if self.state.markers_visible:
 
-            if self.state.cmap_mode == 'Fixed' and self.state.size_mode == 'Fixed':
+            if self.state.density_map:
+
+                if self.state.cmap_mode == 'Fixed':
+                    if force or 'color' in changed or 'cmap_mode' in changed:
+                        self.density_artist.set_color(self.state.color)
+                        self.density_artist.set_c(None)
+                        self.density_artist.set_clim(self.density_auto_limits.min,
+                                                     self.density_auto_limits.max)
+                elif force or any(prop in changed for prop in CMAP_PROPERTIES):
+                    c = self.layer[self.state.cmap_att].ravel()
+                    set_mpl_artist_cmap(self.density_artist, c, self.state)
 
-                if force or 'color' in changed:
-                    self.plot_artist.set_color(self.state.color)
+                if force or 'stretch' in changed:
+                    self.density_artist.set_norm(ImageNormalize(stretch=STRETCHES[self.state.stretch]()))
 
-                if force or 'size' in changed or 'size_scaling' in changed:
-                    self.plot_artist.set_markersize(self.state.size *
-                                                    self.state.size_scaling)
+                if force or 'dpi' in changed:
+                    self.density_artist.set_dpi(self._viewer_state.dpi)
+
+                if force or 'density_contrast' in changed:
+                    self.density_auto_limits.contrast = self.state.density_contrast
+                    self.density_artist.stale = True
 
             else:
 
-                # TEMPORARY: Matplotlib has a bug that causes set_alpha to
-                # change the colors back: https://github.com/matplotlib/matplotlib/issues/8953
-                if 'alpha' in changed:
-                    force = True
+                if self.state.cmap_mode == 'Fixed' and self.state.size_mode == 'Fixed':
 
-                if self.state.cmap_mode == 'Fixed':
-                    if force or 'color' in changed or 'cmap_mode' in changed:
-                        self.scatter_artist.set_facecolors(self.state.color)
+                    if force or 'color' in changed:
+                        self.plot_artist.set_color(self.state.color)
+
+                    if force or 'size' in changed or 'size_scaling' in changed:
+                        self.plot_artist.set_markersize(self.state.size *
+                                                        self.state.size_scaling)
+
+                else:
+
+                    # TEMPORARY: Matplotlib has a bug that causes set_alpha to
+                    # change the colors back: https://github.com/matplotlib/matplotlib/issues/8953
+                    if 'alpha' in changed:
+                        force = True
+
+                    if self.state.cmap_mode == 'Fixed':
+                        if force or 'color' in changed or 'cmap_mode' in changed:
+                            self.scatter_artist.set_facecolors(self.state.color)
+                            self.scatter_artist.set_edgecolor('none')
+                    elif force or any(prop in changed for prop in CMAP_PROPERTIES):
+                        c = self.layer[self.state.cmap_att].ravel()
+                        set_mpl_artist_cmap(self.scatter_artist, c, self.state)
                         self.scatter_artist.set_edgecolor('none')
-                elif force or any(prop in changed for prop in CMAP_PROPERTIES):
-                    c = self.layer[self.state.cmap_att].ravel()
-                    set_mpl_artist_cmap(self.scatter_artist, c, self.state)
-                    self.scatter_artist.set_edgecolor('none')
 
-                if force or any(prop in changed for prop in MARKER_PROPERTIES):
+                    if force or any(prop in changed for prop in MARKER_PROPERTIES):
 
-                    if self.state.size_mode == 'Fixed':
-                        s = self.state.size * self.state.size_scaling
-                        s = broadcast_to(s, self.scatter_artist.get_sizes().shape)
-                    else:
-                        s = self.layer[self.state.size_att].ravel()
-                        s = ((s - self.state.size_vmin) /
-                             (self.state.size_vmax - self.state.size_vmin)) * 30
-                        s *= self.state.size_scaling
+                        if self.state.size_mode == 'Fixed':
+                            s = self.state.size * self.state.size_scaling
+                            s = broadcast_to(s, self.scatter_artist.get_sizes().shape)
+                        else:
+                            s = self.layer[self.state.size_att].ravel()
+                            s = ((s - self.state.size_vmin) /
+                                 (self.state.size_vmax - self.state.size_vmin)) * 30
+                            s *= self.state.size_scaling
 
-                    # Note, we need to square here because for scatter, s is actually
-                    # proportional to the marker area, not radius.
-                    self.scatter_artist.set_sizes(s ** 2)
+                        # Note, we need to square here because for scatter, s is actually
+                        # proportional to the marker area, not radius.
+                        self.scatter_artist.set_sizes(s ** 2)
 
         if self.state.line_visible:
 
@@ -310,7 +372,8 @@ def _update_visual_attributes(self, changed, force=False):
                     eartist.set_zorder(self.state.zorder)
 
         for artist in [self.scatter_artist, self.plot_artist,
-                       self.vector_artist, self.line_collection]:
+                       self.vector_artist, self.line_collection,
+                       self.density_artist]:
 
             if artist is None:
                 continue

glue/viewers/scatter/qt/data_viewer.py

@@ -42,7 +42,7 @@ def _update_axes(self, *args):
         if self.state.x_att is not None:
 
             # Update ticks, which sets the labels to categories if components are categorical
-            update_ticks(self.axes, 'x', self.state._get_x_components(), False)
+            update_ticks(self.axes, 'x', self.state._get_x_components(), self.state.x_log)
 
             if self.state.x_log:
                 self.axes.set_xlabel('Log ' + self.state.x_att.label)
@@ -52,7 +52,7 @@ def _update_axes(self, *args):
         if self.state.y_att is not None:
 
             # Update ticks, which sets the labels to categories if components are categorical
-            update_ticks(self.axes, 'y', self.state._get_y_components(), False)
+            update_ticks(self.axes, 'y', self.state._get_y_components(), self.state.y_log)
 
             if self.state.y_log:
                 self.axes.set_ylabel('Log ' + self.state.y_att.label)