Issue 341 (#342)

* Fixes #341

* Residual py3 issues in DDFacet based predictor

Author: ratt-priv-ci

cubical/data_handler/DDFacetSim.py

@@ -11,7 +11,6 @@
         import DDFacet.cbuild.Gridder._pyGridderSmearPolsClassic3x as _pyGridderSmearPolsClassic
     else:
         import DDFacet.cbuild.Gridder._pyGridderSmearPolsClassic27 as _pyGridderSmearPolsClassic
-    from DDFacet.cbuild.Gridder import _pyGridderSmearPolsClassic27
     from DDFacet.ToolsDir.ModToolBox import EstimateNpix
     from DDFacet.Array import shared_dict
     from DDFacet.ToolsDir import ModFFTW
@@ -138,6 +137,9 @@ def init_sems(cls, NSemaphores = 3373):
     @classmethod
     def del_sems(cls):
         """ Deinit semaphores """
+        if not cls.__should_init_sems:
+            return
+        cls.__should_init_sems = True
         _pyGridderSmearPolsClassic.pyDeleteSemaphore()
         cls.__degridding_semaphores = None
 
@@ -165,7 +167,7 @@ def __cachename_compute(self, src):
                     " scale:" + "x".join(map(str, list(np.deg2rad(src.get_direction_pxoffset(subregion_index=subregion_index)) *
                                                        src.pixel_scale / 3600.0)))
         res = "dir_{0:s}_{1:s}_{2:s}".format(str(self.__model), str(self.__direction), str(reg_props))
-        return hashlib.md5(res).hexdigest()
+        return hashlib.md5(res.encode()).hexdigest()
 
     def __init_grid_machine(self, src, dh, tile, poltype, freqs):
         """ initializes a grid machine for this direction """
@@ -349,9 +351,9 @@ def simulate(self, dh, tile, tile_subset, poltype, uvwco, freqs, model_type):
 
 import atexit
 
-def _cleanup_degridders():
+def cleanup_degridders():
     DDFacetSim.del_sems()
     DDFacetSim.dealloc_degridders()
     DDFacetSim.shutdown_pool()
 
-atexit.register(_cleanup_degridders)
+atexit.register(cleanup_degridders)

cubical/data_handler/DicoSourceProvider.py

@@ -7,7 +7,7 @@
 
 from regions import DS9Parser
 import numpy as np
-from geometry import BoundingConvexHull, BoundingBox, BoundingBoxFactory
+from .geometry import BoundingConvexHull, BoundingBox, BoundingBoxFactory
 from cubical.tools import logger, ModColor
 log = logger.getLogger("DDFacetSim")
 
@@ -108,7 +108,7 @@ def __read_regions_file(self, fn, padding_factor, max_size, min_nfacet_per_axis)
             with open(fn) as f:
                 parser = DS9Parser(f.read())
                 for regi, reg in enumerate(parser.shapes):
-                    coords = map(int, [c.value for c in reg.coord])
+                    coords = list(map(int, [c.value for c in reg.coord]))
                     assert len(coords) % 2 == 0, "Number of region coords must be multiple of 2-tuple"
                     coords = np.array(coords).reshape([len(coords) // 2, 2])
                     clusters.append(BoundingConvexHull(coords,
@@ -133,19 +133,19 @@ def __split_regular_region(reg, max_size):
         log(2).print("\tSplitting regions into facetted regions, with maximum unpadded size of {0:.2f} degrees per facet".format(max_size))
         clusters = [aasubreg for aareg in map(lambda reg: __split_regular_region(reg, max_size), clusters) 
                     for aasubreg in aareg]
-        clusters = map(lambda reg: BoundingBoxFactory.AxisAlignedBoundingBox(reg, square=True, check_mask_outofbounds=False), clusters)
+        clusters = list(map(lambda reg: BoundingBoxFactory.AxisAlignedBoundingBox(reg, square=True, check_mask_outofbounds=False), clusters))
 
         def __pad_cluster(c, padding_factor):
             npx,_ = c.box_npx # square facet at this point
             # this returns an odd npix:
             npixunpadded, npixpadded = EstimateNpix(npx, Padding=padding_factor)
             return BoundingBoxFactory.PadBox(c, npixpadded, npixpadded, check_mask_outofbounds=False)
         log(2).print("\tPadding all facets by a minimum factor of {0:.2f}x".format(padding_factor))
-        clusters = map(lambda c: __pad_cluster(c, padding_factor), clusters)
+        clusters = list(map(lambda c: __pad_cluster(c, padding_factor), clusters))
         log.debug("\tNormalizing regional weights")
         BoundingConvexHull.normalize_masks(clusters)
         log(2).print("\tCaching regional weight maps for future predicts")
-        map(lambda x: x.mask, clusters) # cache mask
+        list(map(lambda x: x.mask, clusters)) # cache mask
         dirs = {} 
         for c in clusters:
             dirs[c.name] = dirs.get(c.name, []) + [c]

cubical/data_handler/ms_tile.py

@@ -1363,6 +1363,8 @@ def release(self):
             if subset.label is not None:
                 data = shared_dict.attach(subset.datadict)
                 data.delete()
+        from .DDFacetSim import cleanup_degridders
+        cleanup_degridders()