From 5a5f8fa9aea9241353f0e439050fa1d755db22aa Mon Sep 17 00:00:00 2001 From: allyson Date: Tue, 23 Jul 2024 15:34:06 -0700 Subject: [PATCH 1/9] pass max_velo_diff arg to deltachi2 comparison --- py/redrock/zfind.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/py/redrock/zfind.py b/py/redrock/zfind.py index 70349f6..324b211 100644 --- a/py/redrock/zfind.py +++ b/py/redrock/zfind.py @@ -110,7 +110,8 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): zwarn : array of zwarn values Options: - dvlimit: exclude candidates that are closer than dvlimit [km/s] + dvlimit: exclude candidates that are closer than dvlimit [km/s], + uses max value of the pair Returns (deltachi2, setzwarn) where `deltachi2` is array of chi2 differences to next best good fit, and `setzwarn` is boolean array of whether @@ -123,9 +124,14 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): nz = len(chi2) deltachi2 = np.zeros(nz) okfit = (zwarn & badfit_mask) == 0 + + # if no dvlimit passed, use constant threshold + if np.isscalar(dvlimit): + dvlimit = np.full(len(chi2), dvlimit) + for i in range(len(chi2)-1): dv = get_dv(z[i+1:], z[i]) - ii = (np.abs(dv)>dvlimit) & okfit[i+1:] + ii = (np.abs(dv)>np.max(dvlimit[i],dvlimit[i+1:])) & okfit[i+1:] if np.any(ii): dchi2 = chi2[i+1:] - chi2[i] deltachi2[i] = np.min(dchi2[ii]) @@ -139,9 +145,10 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): noti[i] = False alldeltachi2 = np.absolute(chi2[noti] - chi2[i]) alldv = np.absolute(get_dv(z=z[noti], zref=z[i])) + alldvlimit = np.maximum(dvlimit[i], dvlimit[noti]) zwarn = np.any( okfit[noti] & (alldeltachi2 < constants.min_deltachi2) & - (alldv >= dvlimit) ) + (alldv >= alldvlimit) ) if zwarn: setzwarn[i] = True @@ -476,13 +483,6 @@ def zfind(targets, templates, mp_procs=1, nminima=3, archetypes=None, priors=Non else: spectype, subtype = parse_fulltype(fulltype) - # set max_velo_diff differently for STARs, but watch out - # for archtypes which have spectype as list instead of scalar - if (np.isscalar(spectype) and spectype.upper() == 'STAR') or spectype[0].upper() == 'STAR': - max_velo_diff = constants.max_velo_diff_star - else: - max_velo_diff = constants.max_velo_diff - #Have to create arrays of correct length since using dict of #np arrays instead of astropy Table nmin = len(tmp['chi2']) @@ -496,6 +496,15 @@ def zfind(targets, templates, mp_procs=1, nminima=3, archetypes=None, priors=Non tmp['subtype'] = np.array([subtype]).reshape((nmin, 1)) tmp['ncoeff'] = np.array([tmp['coeff'].shape[1]]*nmin).reshape((nmin, 1)) + + # set max_velo_diff differently for STARs, but watch out + # for archtypes which have spectype as list instead of scalar + if (np.isscalar(spectype) and spectype.upper() == 'STAR') or spectype[0].upper() == 'STAR': + max_velo_diff = constants.max_velo_diff_star + else: + max_velo_diff = constants.max_velo_diff + tmp['max_velo_diff'] = np.full((nmin,1), max_velo_diff) + tzfit.append(tmp) del allresults[tid][fulltype]['zfit'] @@ -570,10 +579,12 @@ def zfind(targets, templates, mp_procs=1, nminima=3, archetypes=None, priors=Non #- calc deltachi2 and set ZW.SMALL_DELTA_CHI2 flag deltachi2, setzwarn = calc_deltachi2(tzfit['chi2'], tzfit['z'], tzfit['zwarn'], - dvlimit=max_velo_diff) + dvlimit=tzfit['max_velo_diff']) tzfit['deltachi2'] = deltachi2 tzfit['zwarn'][setzwarn] |= ZW.SMALL_DELTA_CHI2 + # remove max_velo_diff_column here? + # Store # Here convert to astropy table allzfit.append(astropy.table.Table(tzfit)) From 6e5c1aeda55cc7b6ea4f6541af36424967f1167f Mon Sep 17 00:00:00 2001 From: allyson Date: Tue, 23 Jul 2024 16:24:30 -0700 Subject: [PATCH 2/9] convert dvlimit to array --- py/redrock/zfind.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/py/redrock/zfind.py b/py/redrock/zfind.py index 324b211..7859f13 100644 --- a/py/redrock/zfind.py +++ b/py/redrock/zfind.py @@ -121,13 +121,14 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): what the next chi2 would have been. This can also occur for the last N targets if all N of them are within dvlimit of each other. ''' + print(chi2,z,zwarn,dvlimit) nz = len(chi2) deltachi2 = np.zeros(nz) okfit = (zwarn & badfit_mask) == 0 # if no dvlimit passed, use constant threshold if np.isscalar(dvlimit): - dvlimit = np.full(len(chi2), dvlimit) + dvlimit = np.full(nz, dvlimit) for i in range(len(chi2)-1): dv = get_dv(z[i+1:], z[i]) @@ -504,7 +505,7 @@ def zfind(targets, templates, mp_procs=1, nminima=3, archetypes=None, priors=Non else: max_velo_diff = constants.max_velo_diff tmp['max_velo_diff'] = np.full((nmin,1), max_velo_diff) - + tzfit.append(tmp) del allresults[tid][fulltype]['zfit'] From b876acde2e26e7a77e26124433889378d1955c8d Mon Sep 17 00:00:00 2001 From: allyson Date: Tue, 23 Jul 2024 17:08:14 -0700 Subject: [PATCH 3/9] delete velo_diff column --- py/redrock/zfind.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/redrock/zfind.py b/py/redrock/zfind.py index 7859f13..49d69a5 100644 --- a/py/redrock/zfind.py +++ b/py/redrock/zfind.py @@ -121,7 +121,6 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): what the next chi2 would have been. This can also occur for the last N targets if all N of them are within dvlimit of each other. ''' - print(chi2,z,zwarn,dvlimit) nz = len(chi2) deltachi2 = np.zeros(nz) okfit = (zwarn & badfit_mask) == 0 @@ -132,7 +131,7 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): for i in range(len(chi2)-1): dv = get_dv(z[i+1:], z[i]) - ii = (np.abs(dv)>np.max(dvlimit[i],dvlimit[i+1:])) & okfit[i+1:] + ii = (np.abs(dv)>np.maximum(dvlimit[i],dvlimit[i+1:])) & okfit[i+1:] if np.any(ii): dchi2 = chi2[i+1:] - chi2[i] deltachi2[i] = np.min(dchi2[ii]) @@ -584,7 +583,8 @@ def zfind(targets, templates, mp_procs=1, nminima=3, archetypes=None, priors=Non tzfit['deltachi2'] = deltachi2 tzfit['zwarn'][setzwarn] |= ZW.SMALL_DELTA_CHI2 - # remove max_velo_diff_column here? + # remove max_velo_diff column + del tzfit['max_velo_diff'] # Store # Here convert to astropy table From 45dde972934cfb8d1fde0d854d6a3ffa1f4c45d0 Mon Sep 17 00:00:00 2001 From: John Moustakas Date: Tue, 2 Jul 2024 15:59:55 -0700 Subject: [PATCH 4/9] add asdict optional keywork to load_templates_from_header --- py/redrock/templates.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/redrock/templates.py b/py/redrock/templates.py index cc6d855..d3f83bb 100644 --- a/py/redrock/templates.py +++ b/py/redrock/templates.py @@ -404,7 +404,7 @@ def header2templatefiles(hdr, template_dir=None): return filenames -def load_templates_from_header(hdr, template_dir=None): +def load_templates_from_header(hdr, template_dir=None, asdict=False): """Return templates matching header keywords Args: @@ -412,11 +412,12 @@ def load_templates_from_header(hdr, template_dir=None): Options: template_dir (str): full path to redrock-templates directory - + asdict (bool): return dict keyed by (spectype, subtype) instead of list + Returns list of Template objects """ template_filenames = header2templatefiles(hdr, template_dir=template_dir) - return load_templates(template_filenames) + return load_templates(template_filenames, asdict=asdict) def load_templates(template_path=None, zscan_galaxy=None, zscan_star=None, zscan_qso=None, asdict=False): From 7277ec3da51904693c5533b11ac238638cefab92 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 18 Jul 2024 13:40:37 -0700 Subject: [PATCH 5/9] remove environment variables --- etc/redrock.module | 6 ------ 1 file changed, 6 deletions(-) diff --git a/etc/redrock.module b/etc/redrock.module index a32c2e4..228be7d 100644 --- a/etc/redrock.module +++ b/etc/redrock.module @@ -76,9 +76,3 @@ setenv [string toupper $product] $PRODUCT_DIR # # Add any non-standard Module code below this point. # -# MPI or multiprocessing handle parallelism, not OpenMP via numpy/MKL -# -setenv KMP_AFFINITY disabled -setenv MPICH_GNI_FORK_MODE FULLCOPY -setenv OMP_NUM_THREADS 1 -setenv HDF5_USE_FILE_LOCKING FALSE From dfe22005f541744bae318385549e82b879d905c3 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 18 Jul 2024 13:48:06 -0700 Subject: [PATCH 6/9] pin numpy version --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2197f55..748dfb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ # Based on desimodules/22.2. +numpy<2.0 astropy scipy h5py From cc41ef44e38709c493df52772fa2a7ed6f76b4e1 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Fri, 19 Jul 2024 09:45:51 -0700 Subject: [PATCH 7/9] update changes.rst for PR 313 --- doc/changes.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/changes.rst b/doc/changes.rst index 21e32fd..d5ce888 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,7 +5,9 @@ redrock Change Log 0.20.2 (unreleased) ------------------- -* No changes yet. +* Module file cleanup env vars moved to desimodules (PR `#313`_). + +.. _`#313`: https://github.com/desihub/redrock/pull/313 0.20.1 (2024-06-04) ------------------- From c606e365b134ada04677b8e3346ba77fee8a5538 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 24 Jul 2024 15:41:33 -0700 Subject: [PATCH 8/9] also write header keywords to REDSHIFTS HDU --- py/redrock/external/desi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py/redrock/external/desi.py b/py/redrock/external/desi.py index c43bc7d..6b0df9d 100644 --- a/py/redrock/external/desi.py +++ b/py/redrock/external/desi.py @@ -107,8 +107,10 @@ def write_zbest(outfile, zbest, fibermap, exp_fibermap, tsnr2, Modifies input tables.meta['EXTNAME'] """ + # header keywords for both PRIMARY and REDSHIFTS HDU header = _get_header(templates, archetypes, spec_header) + zbest.meta.update(header) zbest.meta['EXTNAME'] = 'REDSHIFTS' fibermap.meta['EXTNAME'] = 'FIBERMAP' exp_fibermap.meta['EXTNAME'] = 'EXP_FIBERMAP' From 8adaec1199f740e0fe1c90bc14692283db4f79c7 Mon Sep 17 00:00:00 2001 From: allyson Date: Fri, 26 Jul 2024 14:09:19 -0700 Subject: [PATCH 9/9] uses minimum max_velo_diff of pair when ranking N best fits --- py/redrock/zfind.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/redrock/zfind.py b/py/redrock/zfind.py index 49d69a5..bd2cde1 100644 --- a/py/redrock/zfind.py +++ b/py/redrock/zfind.py @@ -111,7 +111,7 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): Options: dvlimit: exclude candidates that are closer than dvlimit [km/s], - uses max value of the pair + uses minimum value of the pair Returns (deltachi2, setzwarn) where `deltachi2` is array of chi2 differences to next best good fit, and `setzwarn` is boolean array of whether @@ -131,7 +131,7 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): for i in range(len(chi2)-1): dv = get_dv(z[i+1:], z[i]) - ii = (np.abs(dv)>np.maximum(dvlimit[i],dvlimit[i+1:])) & okfit[i+1:] + ii = (np.abs(dv)>np.minimum(dvlimit[i],dvlimit[i+1:])) & okfit[i+1:] if np.any(ii): dchi2 = chi2[i+1:] - chi2[i] deltachi2[i] = np.min(dchi2[ii]) @@ -145,7 +145,7 @@ def calc_deltachi2(chi2, z, zwarn, dvlimit=constants.max_velo_diff): noti[i] = False alldeltachi2 = np.absolute(chi2[noti] - chi2[i]) alldv = np.absolute(get_dv(z=z[noti], zref=z[i])) - alldvlimit = np.maximum(dvlimit[i], dvlimit[noti]) + alldvlimit = np.minimum(dvlimit[i], dvlimit[noti]) zwarn = np.any( okfit[noti] & (alldeltachi2 < constants.min_deltachi2) & (alldv >= alldvlimit) )