From 9d38e2db1c01c4dd9907f7ca22d5a26687649fc3 Mon Sep 17 00:00:00 2001 From: Martin-Rey Date: Fri, 14 Dec 2018 17:35:42 +0000 Subject: [PATCH 1/6] First shot at implementing a fix for the contamination fraction in server mode --- tangos/input_handlers/pynbody.py | 28 ++++++++++++++++++++++------ tangos/properties/pynbody/zoom.py | 21 +++++++++++++++++---- tests/test_simulation_outputs.py | 8 ++++++++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/tangos/input_handlers/pynbody.py b/tangos/input_handlers/pynbody.py index 97a900e3..0cc94f35 100644 --- a/tangos/input_handlers/pynbody.py +++ b/tangos/input_handlers/pynbody.py @@ -246,16 +246,17 @@ def get_properties(self): if len(timesteps)>0: f = self.load_timestep_without_caching(sorted(timesteps)[-1]) if self.quicker: - res = self._estimate_resolution_quicker(f) + res_kpc = self._estimate_spatial_resolution_quicker(f) + res_msol = self._estimate_mass_resolution_quicker(f) else: - res = self._estimate_resolution(f) - - return {'approx_resolution_kpc': res} + res_kpc = self._estimate_spatial_resolution(f) + res_msol = self._estimate_mass_resolution(f) + return {'approx_resolution_kpc': res_kpc, 'approx_resolution_Msol': res_msol} else: return {} - def _estimate_resolution(self, f): + def _estimate_spatial_resolution(self, f): f.physical_units() if "eps" in f.dm.loadable_keys(): # Interpret the eps array as a softening, and assume that it is not a comoving softening (as the @@ -273,7 +274,7 @@ def _estimate_resolution(self, f): estimated_eps = 0.01 * frac_length * f.properties['boxsize'].in_units('kpc a', **f.conversion_context()) return float(estimated_eps) - def _estimate_resolution_quicker(self, f): + def _estimate_spatial_resolution_quicker(self, f): interparticle_distance = float(f.properties['boxsize'].in_units("kpc a",**f.conversion_context()))/(float(len(f))**(1./3)) res = 0.01*interparticle_distance logger.warn("Because 'quicker' flag is set, estimating res %.2g kpc from the file size; this could be inaccurate", @@ -281,6 +282,21 @@ def _estimate_resolution_quicker(self, f): logger.warn(" -- it will certainly be wrong for e.g. zoom simulations") return res + def _estimate_mass_resolution(self, f): + f.physical_units() + if "mass" in f.dm.loadable_keys(): + min_simulation_mass = f.dm['mass'].min() + return float(min_simulation_mass) + + def _estimate_mass_resolution_quicker(self, f): + import pynbody.analysis.cosmology as cosmo + rho_m = cosmo.rho_M(f, z=0, unit="Msol kpc**-3 a**-3") + volume_box = float(f.properties['boxsize'].in_units("kpc a",**f.conversion_context()) ** 3) + estimated_part_mass = rho_m * volume_box / len(f) + logger.warn("Because 'quicker' flag is set, estimating mass res %.2g msol from the file size; this could be inaccurate", + estimated_part_mass) + logger.warn(" -- it will certainly be wrong for e.g. zoom simulations") + return estimated_part_mass class RamsesHOPInputHandler(PynbodyInputHandler): patterns = ["output_0????"] diff --git a/tangos/properties/pynbody/zoom.py b/tangos/properties/pynbody/zoom.py index 04bd07b0..306eff17 100644 --- a/tangos/properties/pynbody/zoom.py +++ b/tangos/properties/pynbody/zoom.py @@ -1,12 +1,25 @@ from . import PynbodyPropertyCalculation + class Contamination(PynbodyPropertyCalculation): + """ Calculates the contamination of a zoomed halo by heavier, unzoomed particles. + The current behaviour is to return 1.0 for halo containing no particles of the deepest zoom level, + the mass fraction of heavy to light particles for halos containing a mixture of particle masses + and 0.0 for halos containing exclusively particles from the deepest zoom level. + """ + + names = "contamination_fraction" def calculate(self, halo, exist): - n_heavy = (halo.dm['mass'] > self.min_dm_mass).sum() - return float(n_heavy) / len(halo.dm) + simulation_min_dm_mass = self._simulation.get("approx_resolution_Msol", 0.1) + if self.loaded_data_min_dm_mass > simulation_min_dm_mass: + # Loaded data contains only "heavy" particles, e.g. an unzoomed halo in server mode + return 1.0 + else: + # Loaded data contains heavy and/or light particles, e.g + n_heavy = (halo.dm['mass'] > self.loaded_data_min_dm_mass).sum() + return float(n_heavy) / len(halo.dm) def preloop(self, sim, db_timestep): - self.min_dm_mass = sim.dm['mass'].min() + self.loaded_data_min_dm_mass = sim.dm['mass'].min() - names = "contamination_fraction" \ No newline at end of file diff --git a/tests/test_simulation_outputs.py b/tests/test_simulation_outputs.py index 155dbb63..310cbddd 100644 --- a/tests/test_simulation_outputs.py +++ b/tests/test_simulation_outputs.py @@ -25,6 +25,14 @@ def test_get_deprecated_handler(): def test_handler_name(): assert pynbody_outputs.ChangaInputHandler.handler_class_name()=="pynbody.ChangaInputHandler" +def test_handler_properties(): + prop = output_manager.get_properties() + assert len(prop) == 10 + assert prop.__contains__('approx_resolution_kpc') + assert prop.__contains__('approx_resolution_Msol') + npt.assert_allclose(prop['approx_resolution_kpc'], 0.3499348849) + npt.assert_allclose(prop['approx_resolution_Msol'], 144411.17640) + def test_enumerate(): assert set(output_manager.enumerate_timestep_extensions())==set(["tiny.000640","tiny.000832"]) From bd55609d45f9e41b123652b8b471c126ac81389d Mon Sep 17 00:00:00 2001 From: Martin-Rey Date: Tue, 22 Jan 2019 18:06:10 +0000 Subject: [PATCH 2/6] Clearing up tested contamination_fraction and adding test for quicker mode --- tangos/properties/pynbody/zoom.py | 20 ++++++++++---------- tests/test_simulation_outputs.py | 7 ++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tangos/properties/pynbody/zoom.py b/tangos/properties/pynbody/zoom.py index 306eff17..9b57d481 100644 --- a/tangos/properties/pynbody/zoom.py +++ b/tangos/properties/pynbody/zoom.py @@ -3,23 +3,23 @@ class Contamination(PynbodyPropertyCalculation): """ Calculates the contamination of a zoomed halo by heavier, unzoomed particles. - The current behaviour is to return 1.0 for halo containing no particles of the deepest zoom level, - the mass fraction of heavy to light particles for halos containing a mixture of particle masses - and 0.0 for halos containing exclusively particles from the deepest zoom level. + The current behaviour returns: + 1.0 if the halo contains only heavy particles + X.X if the halo contains a mixture of heavy and light particles + 0.0 if the halo contains exclusively light particles + Heavies are defined as any particle heavier than the deepest zoom level if multiple zooms are performed """ names = "contamination_fraction" def calculate(self, halo, exist): - simulation_min_dm_mass = self._simulation.get("approx_resolution_Msol", 0.1) - if self.loaded_data_min_dm_mass > simulation_min_dm_mass: + loaded_data_min_dm_mass = halo.dm['mass'].min() + simulation_min_dm_mass = self._simulation.get("approx_resolution_Msol") + + if loaded_data_min_dm_mass > simulation_min_dm_mass: # Loaded data contains only "heavy" particles, e.g. an unzoomed halo in server mode return 1.0 else: # Loaded data contains heavy and/or light particles, e.g - n_heavy = (halo.dm['mass'] > self.loaded_data_min_dm_mass).sum() + n_heavy = (halo.dm['mass'] > loaded_data_min_dm_mass).sum() return float(n_heavy) / len(halo.dm) - - def preloop(self, sim, db_timestep): - self.loaded_data_min_dm_mass = sim.dm['mass'].min() - diff --git a/tests/test_simulation_outputs.py b/tests/test_simulation_outputs.py index 310cbddd..204ad2d6 100644 --- a/tests/test_simulation_outputs.py +++ b/tests/test_simulation_outputs.py @@ -15,7 +15,6 @@ def setup(): db.config.base = os.path.join(os.path.dirname(__file__), "test_simulations") output_manager = pynbody_outputs.ChangaInputHandler("test_tipsy") - def test_get_handler(): assert db.input_handlers.get_named_handler_class('pynbody.ChangaInputHandler') == pynbody_outputs.ChangaInputHandler @@ -33,6 +32,12 @@ def test_handler_properties(): npt.assert_allclose(prop['approx_resolution_kpc'], 0.3499348849) npt.assert_allclose(prop['approx_resolution_Msol'], 144411.17640) +def test_handler_properties_quicker_flag(): + output_manager.quicker = True + prop = output_manager.get_properties() + npt.assert_allclose(prop['approx_resolution_kpc'], 33.4360203909648) + npt.assert_allclose(prop['approx_resolution_Msol'], 40370039199.44858) + def test_enumerate(): assert set(output_manager.enumerate_timestep_extensions())==set(["tiny.000640","tiny.000832"]) From 06fa21c42763f68da8f4895f96fa70ea365dc345 Mon Sep 17 00:00:00 2001 From: Martin-Rey Date: Wed, 23 Jan 2019 08:39:16 +0000 Subject: [PATCH 3/6] Trying to isolate from which numpy/cython version releases the errors come from --- .travis.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25bb67f1..83589f73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,12 @@ sudo: false # specify different versions of python and numpy env: - - PYTHON=2.7 NUMPY_VERSION=1.10.4 - - PYTHON=2.7 NUMPY_VERSION=1.13.1 - - PYTHON=3.5 NUMPY_VERSION=1.13.1 - - PYTHON=3.6 NUMPY_VERSION=1.13.1 + - PYTHON=2.7 NUMPY_VERSION=1.10.4 CYTHON_VERSION=0.28 + - PYTHON=2.7 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 + - PYTHON=3.5 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 + - PYTHON=3.6 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 + - PYTHON=3.6 NUMPY_VERSION=1.16.0 CYTHON_VERSION=0.28 + - PYTHON=3.6 NUMPY_VERSION=1.16.0 CYTHON_VERSION=0.29 before_install: - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh @@ -20,7 +22,7 @@ before_install: - conda update --yes conda install: - - conda create --yes -n ${PYTHON}_${NUMPY_VERSION} python=$PYTHON numpy=$NUMPY_VERSION scipy matplotlib nose h5py pip webtest pyquery sphinx pygments pandas sqlalchemy cython pyramid pyramid_mako ipython + - conda create --yes -n ${PYTHON}_${NUMPY_VERSION} python=$PYTHON numpy=$NUMPY_VERSION scipy matplotlib nose h5py pip webtest pyquery sphinx pygments pandas sqlalchemy cython=$CYTHON_VERSION pyramid pyramid_mako ipython - source activate ${PYTHON}_${NUMPY_VERSION} - python setup.py install - python -c "import tangos" # ensure tangos can be imported without pynbody present From 04beb9c83b0e70114f0c359dd1ba86397a59d8b2 Mon Sep 17 00:00:00 2001 From: Martin-Rey Date: Wed, 23 Jan 2019 08:47:44 +0000 Subject: [PATCH 4/6] Reducing numpy versions not available in pip yet --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 83589f73..55777664 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,9 @@ env: - PYTHON=2.7 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 - PYTHON=3.5 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 - PYTHON=3.6 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 - - PYTHON=3.6 NUMPY_VERSION=1.16.0 CYTHON_VERSION=0.28 - - PYTHON=3.6 NUMPY_VERSION=1.16.0 CYTHON_VERSION=0.29 + - PYTHON=3.6 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.29 + - PYTHON=3.6 NUMPY_VERSION=1.15.0 CYTHON_VERSION=0.28 + - PYTHON=3.6 NUMPY_VERSION=1.15.0 CYTHON_VERSION=0.29 before_install: - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh From 91ce8bb66efe5e12b2bcc5171072d8065997c44e Mon Sep 17 00:00:00 2001 From: Martin-Rey Date: Wed, 23 Jan 2019 09:18:51 +0000 Subject: [PATCH 5/6] Reverting the travis builds since changing numpy/cython versions did not yield any results --- .travis.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55777664..25bb67f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,10 @@ sudo: false # specify different versions of python and numpy env: - - PYTHON=2.7 NUMPY_VERSION=1.10.4 CYTHON_VERSION=0.28 - - PYTHON=2.7 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 - - PYTHON=3.5 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 - - PYTHON=3.6 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.28 - - PYTHON=3.6 NUMPY_VERSION=1.13.1 CYTHON_VERSION=0.29 - - PYTHON=3.6 NUMPY_VERSION=1.15.0 CYTHON_VERSION=0.28 - - PYTHON=3.6 NUMPY_VERSION=1.15.0 CYTHON_VERSION=0.29 + - PYTHON=2.7 NUMPY_VERSION=1.10.4 + - PYTHON=2.7 NUMPY_VERSION=1.13.1 + - PYTHON=3.5 NUMPY_VERSION=1.13.1 + - PYTHON=3.6 NUMPY_VERSION=1.13.1 before_install: - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh @@ -23,7 +20,7 @@ before_install: - conda update --yes conda install: - - conda create --yes -n ${PYTHON}_${NUMPY_VERSION} python=$PYTHON numpy=$NUMPY_VERSION scipy matplotlib nose h5py pip webtest pyquery sphinx pygments pandas sqlalchemy cython=$CYTHON_VERSION pyramid pyramid_mako ipython + - conda create --yes -n ${PYTHON}_${NUMPY_VERSION} python=$PYTHON numpy=$NUMPY_VERSION scipy matplotlib nose h5py pip webtest pyquery sphinx pygments pandas sqlalchemy cython pyramid pyramid_mako ipython - source activate ${PYTHON}_${NUMPY_VERSION} - python setup.py install - python -c "import tangos" # ensure tangos can be imported without pynbody present From cdeb3fd9bb563c2ea4f82751d253eccb12ffe5e7 Mon Sep 17 00:00:00 2001 From: Martin-Rey Date: Thu, 21 Feb 2019 09:54:13 +0000 Subject: [PATCH 6/6] Adding a default value in case mass resolution is not present and getting rid of __contains__ calls --- tangos/properties/pynbody/zoom.py | 5 ++++- tests/test_simulation_outputs.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tangos/properties/pynbody/zoom.py b/tangos/properties/pynbody/zoom.py index 9b57d481..80007b49 100644 --- a/tangos/properties/pynbody/zoom.py +++ b/tangos/properties/pynbody/zoom.py @@ -14,7 +14,10 @@ class Contamination(PynbodyPropertyCalculation): def calculate(self, halo, exist): loaded_data_min_dm_mass = halo.dm['mass'].min() - simulation_min_dm_mass = self._simulation.get("approx_resolution_Msol") + + # If mass resolution not present as a property, ensure backward compatibility by setting the min mass to infinity + import numpy as np + simulation_min_dm_mass = self._simulation.get("approx_resolution_Msol", default=np.inf) if loaded_data_min_dm_mass > simulation_min_dm_mass: # Loaded data contains only "heavy" particles, e.g. an unzoomed halo in server mode diff --git a/tests/test_simulation_outputs.py b/tests/test_simulation_outputs.py index 204ad2d6..e2bc40d0 100644 --- a/tests/test_simulation_outputs.py +++ b/tests/test_simulation_outputs.py @@ -27,8 +27,8 @@ def test_handler_name(): def test_handler_properties(): prop = output_manager.get_properties() assert len(prop) == 10 - assert prop.__contains__('approx_resolution_kpc') - assert prop.__contains__('approx_resolution_Msol') + assert 'approx_resolution_kpc' in prop + assert 'approx_resolution_Msol' in prop npt.assert_allclose(prop['approx_resolution_kpc'], 0.3499348849) npt.assert_allclose(prop['approx_resolution_Msol'], 144411.17640)