From 565fe4ce612bbfb49b148abbdd6e0f13a5932a05 Mon Sep 17 00:00:00 2001 From: ashkelly Date: Sat, 6 Jun 2020 11:55:25 +0100 Subject: [PATCH 1/7] Added extra dont cache case for when a bounding box has been applied --- yt/geometry/particle_geometry_handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 8722ad58e25..6247ac0bf41 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -92,7 +92,7 @@ def _initialize_index(self): ds.domain_left_edge = ds.arr(1.05*min_ppos, 'code_length') ds.domain_right_edge = ds.arr(1.05*max_ppos, 'code_length') ds.domain_width = ds.domain_right_edge - ds.domain_left_edge - + # use a trivial morton index for datasets containing a single chunk if len(self.data_files) == 1: order1 = 1 @@ -106,6 +106,11 @@ def _initialize_index(self): else: dont_cache = False + # If we have applied a bounding box then we can't cache the + # ParticleBitmap because it is doman dependent + if getattr(ds, "domain_override", False): + dont_cache = True + if not hasattr(self.ds, '_file_hash'): self.ds._file_hash = self._generate_hash() From 77b06ed8aeca5803eb0204a42024a499fa5408a5 Mon Sep 17 00:00:00 2001 From: ashkelly Date: Sat, 6 Jun 2020 11:55:59 +0100 Subject: [PATCH 2/7] Added ds.domain_override for cases when a bbox is applied --- yt/frontends/gadget/data_structures.py | 2 ++ yt/frontends/sdf/data_structures.py | 2 ++ yt/frontends/tipsy/data_structures.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/yt/frontends/gadget/data_structures.py b/yt/frontends/gadget/data_structures.py index 3b8b8b21a3b..e8706ff8b2b 100644 --- a/yt/frontends/gadget/data_structures.py +++ b/yt/frontends/gadget/data_structures.py @@ -271,6 +271,8 @@ def __init__(self, filename, dataset_type="gadget_binary", unit_base['cmcm'] = 1.0 / unit_base["UnitLength_in_cm"] self._unit_base = unit_base if bounding_box is not None: + # This ensures that we know a bounding box has been applied + self.domain_override = True bbox = np.array(bounding_box, dtype="float64") if bbox.shape == (2, 3): bbox = bbox.transpose() diff --git a/yt/frontends/sdf/data_structures.py b/yt/frontends/sdf/data_structures.py index 4b53a66ab5f..ac1cbdf748a 100644 --- a/yt/frontends/sdf/data_structures.py +++ b/yt/frontends/sdf/data_structures.py @@ -62,6 +62,8 @@ def __init__(self, filename, dataset_type="sdf_particles", units_override=None, unit_system="cgs"): if bounding_box is not None: + # This ensures that we know a bounding box has been applied + self.domain_override = True self._subspace = True bbox = np.array(bounding_box, dtype="float64") if bbox.shape == (2, 3): diff --git a/yt/frontends/tipsy/data_structures.py b/yt/frontends/tipsy/data_structures.py index bd88b96dbe7..1f3f77ecf7f 100644 --- a/yt/frontends/tipsy/data_structures.py +++ b/yt/frontends/tipsy/data_structures.py @@ -174,6 +174,8 @@ def _parse_parameter_file(self): self.domain_left_edge = None self.domain_right_edge = None else: + # This ensures that we know a bounding box has been applied + self.domain_override = True bbox = np.array(self.bounding_box, dtype="float64") if bbox.shape == (2, 3): bbox = bbox.transpose() From 3cfa46cc040bfc055a88a0f845edbb0ecac04d23 Mon Sep 17 00:00:00 2001 From: Ashley Kelly Date: Tue, 9 Jun 2020 13:44:04 +0100 Subject: [PATCH 3/7] Apply suggestions from code review made domain_override private Co-authored-by: Matthew Turk --- yt/frontends/gadget/data_structures.py | 2 +- yt/frontends/sdf/data_structures.py | 2 +- yt/frontends/tipsy/data_structures.py | 2 +- yt/geometry/particle_geometry_handler.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yt/frontends/gadget/data_structures.py b/yt/frontends/gadget/data_structures.py index e8706ff8b2b..e7547f0ef99 100644 --- a/yt/frontends/gadget/data_structures.py +++ b/yt/frontends/gadget/data_structures.py @@ -272,7 +272,7 @@ def __init__(self, filename, dataset_type="gadget_binary", self._unit_base = unit_base if bounding_box is not None: # This ensures that we know a bounding box has been applied - self.domain_override = True + self._domain_override = True bbox = np.array(bounding_box, dtype="float64") if bbox.shape == (2, 3): bbox = bbox.transpose() diff --git a/yt/frontends/sdf/data_structures.py b/yt/frontends/sdf/data_structures.py index ac1cbdf748a..3651c97c147 100644 --- a/yt/frontends/sdf/data_structures.py +++ b/yt/frontends/sdf/data_structures.py @@ -63,7 +63,7 @@ def __init__(self, filename, dataset_type="sdf_particles", unit_system="cgs"): if bounding_box is not None: # This ensures that we know a bounding box has been applied - self.domain_override = True + self._domain_override = True self._subspace = True bbox = np.array(bounding_box, dtype="float64") if bbox.shape == (2, 3): diff --git a/yt/frontends/tipsy/data_structures.py b/yt/frontends/tipsy/data_structures.py index 1f3f77ecf7f..b0e771ca9bf 100644 --- a/yt/frontends/tipsy/data_structures.py +++ b/yt/frontends/tipsy/data_structures.py @@ -175,7 +175,7 @@ def _parse_parameter_file(self): self.domain_right_edge = None else: # This ensures that we know a bounding box has been applied - self.domain_override = True + self._domain_override = True bbox = np.array(self.bounding_box, dtype="float64") if bbox.shape == (2, 3): bbox = bbox.transpose() diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 6247ac0bf41..e91cb9e8d8a 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -108,7 +108,7 @@ def _initialize_index(self): # If we have applied a bounding box then we can't cache the # ParticleBitmap because it is doman dependent - if getattr(ds, "domain_override", False): + if getattr(ds, "_domain_override", False): dont_cache = True if not hasattr(self.ds, '_file_hash'): From 9fb8ee7510d2fcd389982bc0581f321b1adaf0e1 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Tue, 9 Jun 2020 15:44:44 -0500 Subject: [PATCH 4/7] Add another dont_cache check for loading --- yt/geometry/particle_geometry_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index e91cb9e8d8a..ed4cba885aa 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -128,11 +128,12 @@ def _initialize_index(self): else: fname = ds.index_filename + dont_load = dont_cache and not hasattr(ds, 'index_filename') try: rflag = self.regions.load_bitmasks(fname) rflag = self.regions.check_bitmasks() self._initialize_frontend_specific() - if rflag == 0: + if dont_load or rflag == 0: raise OSError except OSError: self.regions.reset_bitmasks() From bc17bb088ad732b32dd0990a4288570ae86f13b1 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Tue, 9 Jun 2020 16:03:28 -0500 Subject: [PATCH 5/7] Adding test --- yt/frontends/arepo/tests/test_outputs.py | 16 ++++++++++++++-- yt/geometry/particle_geometry_handler.py | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/yt/frontends/arepo/tests/test_outputs.py b/yt/frontends/arepo/tests/test_outputs.py index 8b9e1a4d8c8..54d1b268469 100644 --- a/yt/frontends/arepo/tests/test_outputs.py +++ b/yt/frontends/arepo/tests/test_outputs.py @@ -1,3 +1,5 @@ +import os +import tempfile from collections import OrderedDict from yt.testing import requires_file from yt.utilities.answer_testing.framework import \ @@ -8,6 +10,7 @@ bullet_h5 = "ArepoBullet/snapshot_150.hdf5" tng59_h5 = "TNGHalo/halo_59.hdf5" +_tng59_bbox = [[45135.0, 51343.0], [51844.0, 56184.0], [60555.0, 63451.0]] @requires_file(bullet_h5) @@ -55,12 +58,21 @@ def test_tng_hdf5(): ] ) - @requires_ds(tng59_h5) def test_arepo_tng59(): - ds = data_dir_load(tng59_h5) + ds = data_dir_load(tng59_h5, kwargs = {'bounding_box': _tng59_bbox}) for test in sph_answer(ds, 'halo_59', 10107142, tng59_fields): test_arepo_tng59.__name__ = test.description yield test +@requires_ds(tng59_h5) +def test_index_override(): + # This tests that we can supply an index_filename, and that when we do, it + # doesn't get written if our bounding_box is overwritten. + tmpfd, tmpname = tempfile.mkstemp(suffix=".ewah") + os.close(tmpfd) + ds = data_dir_load(tng59_h5, kwargs = {'index_filename': tmpname, + 'bounding_box': _tng59_bbox}) + ds.index + assert len(open(tmpname, "r").read()) == 0 diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index ed4cba885aa..01bef20ce4f 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -3,6 +3,7 @@ import numpy as np import os import weakref +import struct from yt.funcs import \ get_pbar, \ @@ -135,7 +136,7 @@ def _initialize_index(self): self._initialize_frontend_specific() if dont_load or rflag == 0: raise OSError - except OSError: + except (OSError, struct.error): self.regions.reset_bitmasks() self._initialize_coarse_index() self._initialize_refined_index() From 3f1c8f3bbd20a6d69234a43b4398ae90d48471e3 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Tue, 9 Jun 2020 16:07:07 -0500 Subject: [PATCH 6/7] Updating AREPO test numbers to update bounding box --- tests/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index a05fd93632a..b344558cc93 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -1,5 +1,5 @@ answer_tests: - local_arepo_002: + local_arepo_003: - yt/frontends/arepo/tests/test_outputs.py local_artio_001: From ce427fa4775cee3a0b44536530ff44e9636c0364 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Thu, 18 Jun 2020 10:31:10 -0500 Subject: [PATCH 7/7] Minor fix for aesthetics of log output --- yt/geometry/particle_geometry_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 01bef20ce4f..03ac22e961c 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -131,10 +131,12 @@ def _initialize_index(self): dont_load = dont_cache and not hasattr(ds, 'index_filename') try: + if dont_load: + raise OSError rflag = self.regions.load_bitmasks(fname) rflag = self.regions.check_bitmasks() self._initialize_frontend_specific() - if dont_load or rflag == 0: + if rflag == 0: raise OSError except (OSError, struct.error): self.regions.reset_bitmasks()