Merge pull request #275 from LSSTDESC/u/yymao/photo-calib-reader

photo-z calibration catalog reader

GCRCatalogs/catalog_configs/cosmoDC2_v1.1.4_image_photoz_calib.yaml

@@ -0,0 +1,6 @@
+subclass_name: composite.CompositeReader
+only_use_master_attr: true
+catalogs:
+  - catalog_name: cosmoDC2_v1.1.4_image
+  - catalog_name: pz_calibrate_references_cosmoDC2_v1.1.4_image
+    matching_method: MATCHING_FORMAT

GCRCatalogs/catalog_configs/cosmoDC2_v1.1.4_small_photoz_calib.yaml

@@ -0,0 +1,6 @@
+subclass_name: composite.CompositeReader
+only_use_master_attr: true
+catalogs:
+  - catalog_name: cosmoDC2_v1.1.4_small
+  - catalog_name: pz_calibrate_references_cosmoDC2_v1.1.4_small
+    matching_method: MATCHING_FORMAT

GCRCatalogs/catalog_configs/pz_calibrate_references_cosmoDC2_v1.1.4_image.yaml

@@ -0,0 +1,10 @@
+subclass_name: photoz_calibrate.PZCalibrateCatalog
+base_dir: /global/projecta/projectdirs/lsst/groups/PZ/PhotoZDC2/COSMODC2v1.1.4/PZCalibrate_image
+description: A catalog of booleans for selecting reference redshifts for use in cross-correlation redshifts (PZCalibrate).
+creators: ['Christopher B. Morrison']
+healpix_pixels: [8786, 8787, 8788, 8789, 8790, 8791, 8792, 8793, 8794, 8913, 8914, 8915, 8916, 8917, 8918, 8919, 8920, 8921, 9042, 9043, 9044, 9045, 9046, 9047, 9048, 9049,
+                 9050, 9169, 9170, 9171, 9172, 9173, 9174, 9175, 9176, 9177, 9178, 9298, 9299, 9300, 9301, 9302, 9303, 9304, 9305, 9306, 9425, 9426, 9427, 9428, 9429, 9430,
+                 9431, 9432, 9433, 9434, 9554, 9555, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9681, 9682, 9683, 9684, 9685, 9686, 9687, 9688, 9689, 9690, 9810, 9811, 9812,
+                 9813, 9814, 9815, 9816, 9817, 9818, 9937, 9938, 9939, 9940, 9941, 9942, 9943, 9944, 9945, 9946, 10066, 10067, 10068, 10069, 10070, 10071, 10072, 10073, 10074, 10193,
+                 10194, 10195, 10196, 10197, 10198, 10199, 10200, 10201, 10202, 10321, 10322, 10323, 10324, 10325, 10326, 10327, 10328, 10329, 10444, 10445, 10446, 10447, 10448, 10449, 10450,
+                 10451, 10452]

GCRCatalogs/catalog_configs/pz_calibrate_references_cosmoDC2_v1.1.4_small.yaml

@@ -0,0 +1,6 @@
+subclass_name: photoz_calibrate.PZCalibrateCatalog
+base_dir: /global/projecta/projectdirs/lsst/groups/PZ/PhotoZDC2/COSMODC2v1.1.4/PZCalibrate_image
+description: A catalog of booleans for selecting reference redshifts for use in cross-correlation redshifts (PZCalibrate).
+creators: ['Christopher B. Morrison']
+healpix_pixels: [9559,  9686,  9687,  9814,  9815,  9816,  9942,  9943, 10070,
+                 10071, 10072, 10198, 10199, 10200, 10326, 10327, 10450]

GCRCatalogs/photoz_calibrate.py

@@ -0,0 +1,84 @@
+"""
+PZCalibrate reference objects catalog reader
+
+This reader was designed by Yao-Yuan Mao,
+based a catalog of "spectroscopic" reference objects for use in cross-
+correlation redshifts provided by Chris Morrison, in Mar 2019.
+"""
+
+import re
+import os
+
+import numpy as np
+from GCR import BaseGenericCatalog
+
+from .utils import first
+
+__all__ = ['PZCalibrateCatalog']
+
+FILE_PATTERN = r'z_(\d)\S+healpix_(\d+)_pz_calib\.npz$'
+
+
+class PZCalibrateCatalog(BaseGenericCatalog):
+
+    def _subclass_init(self, **kwargs):
+        self.base_dir = kwargs['base_dir']
+        self._filename_re = re.compile(kwargs.get('filename_pattern', FILE_PATTERN))
+        self._healpix_pixels = kwargs.get('healpix_pixels')
+
+        self._healpix_files = dict()
+        for f in sorted(os.listdir(self.base_dir)):
+            m = self._filename_re.match(f)
+            if m is None:
+                continue
+            key = tuple(map(int, m.groups()))
+            if self._healpix_pixels and key[1] not in self._healpix_pixels:
+                continue
+            self._healpix_files[key] = os.path.join(self.base_dir, f)
+
+        self._native_filter_quantities = {'healpix_pixel', 'redshift_block_lower'}
+
+        self._quantity_dict = {
+            "QSO": "Flag selecting QSOs by BlackHoleMass and EddingtonRatio. Objects have a mag/redshift "
+                   "distributions similar to those in DESI and are meant to be used as reference objects "
+                   "in cross-correlation redshift analyses.",
+            "LRG": "Flag selecting LRGs by stellar mass. Objects have a mag/redshift "
+                   "distributions similar to those in DESI and are meant to be used as reference objects "
+                   "in cross-correlation redshift analyses.",
+            "ELG": "Flag selecting ELGs by star formation rate. Objects have a mag/redshift "
+                   "distributions similar to those in DESI and are meant to be used as reference objects "
+                   "in cross-correlation redshift analyses.",
+            "MagLim": "Flag selection all objects R<19.4. Objects have a mag/redshift "
+                      "distributions similar to those in DESI and are meant to be used as reference objects "
+                      "in cross-correlation redshift analyses.",
+            "AllReferences": "Union of QSO, LRG, ELG, and MagLim flags. Objects have a mag/redshift "
+                             "distributions similar to those in DESI and are meant to be used as reference "
+                             "objects in cross-correlation redshift analyses.",
+        }
+
+        self._quantity_modifiers = {q: q for q in self._quantity_dict}
+
+    def _get_quantity_info_dict(self, quantity, default=None):
+        """Return a dictionary with descriptive information for a quantity
+
+        Returned information includes a quantity description, quantity units, whether
+        the quantity is defined in the DPDD, and if the quantity is available in GCRbase.
+
+        Args:
+            quantity   (str): The quantity to return information for
+            default (object): Value to return if no information is available (default None)
+
+        Returns:
+            String describing the quantity.
+        """
+        return self._quantity_dict.get(quantity, default)
+
+    def _generate_native_quantity_list(self):
+        return list(np.load(first(self._healpix_files.values())).keys())
+
+    def _iter_native_dataset(self, native_filters=None):
+        for (zlo_this, hpx_this), file_path in self._healpix_files.items():
+            d = {'healpix_pixel': hpx_this, 'redshift_block_lower': zlo_this}
+            if native_filters is not None and not native_filters.check_scalar(d):
+                continue
+            yield np.load(file_path).__getitem__