Prepare v2.3.3 release (#4596)

* Use gating windows in sngl_minifollowup and allow 'triggers within vetoes' option (#4549) * Use gating windows in sngl_minifollowup - also allow 'triggers within vetoes' option * Minor fixes * add infor if fewer triggers than requested * Allow pycbc_sngls_minifollowup to use single ranking statistic * Fix bug in rank_stat_single for quadsum/phasetd statistics * mask may be boolean now * fix broken pycbc_page_snglinfo due to bugfix * page_snglinfo to use the ranking statistic used * missed change * Fix cases where ifar limit was not being applied (#4574) * Fix cases where ifar limit was not being applied * some more missed cases / long lines * redoing a couple more long lines * missed one more place * unrelated bug where the IFOs were not in the right order for use with the significance_dict * remove removal of downranked triggers (#4579) * remove removal of downranked triggers * add brief explanation into the caption * bug in choosing the far calculation method (#4593) * bug in which IFO combo is used for calculating the FAR in each combination during HR * Safety catch * Update release number * add coordinates_space.py (#4289) * add coordinates_space.py * add LISA/SSB frame params * add LISA_to_GEO and GEO_to_LISA * add coordinates_space into FieldArray * add doc and Astropy support * update comments on sympy * use fsolve from scipy instead * fix cc issues * fix cc issues * minor fix * update * not use iteration * decouple LISA orbit and more accurate Earth * rename * remove jplephem * add the angular displacement of the Earth * use radians * make func readable in .ini * reverse back to master * correct psi range * reverse to master * fix unit issue in earth_position_SSB * put LISA to the "right" position * add LISA specific transform classes here * change names * update * make a package for coordinates * remove coordinates_space import * move __all__ into __init__.py * remove all coordinates_space * change TIME_OFFSET to seconds * fix SOBHB issue * rename * add SSB or LISA params into fid_params * rename * fix cc issues * fix cc issue * fix cc issue * update * update * fix * add default names * overwrite params with same names * remove pre-fixed names * remove all pre-fixed names * not pop * fix inverse transform * update tc * not overwrite * add SNR support for multi-model * Update waveform.py * t0 issue * t0 issue * Update space.py * add obstime * np.mod(psi_newframe, 2*np.pi) * fix obstime * add support for array inputs * Update hierarchical.py * just use Alex's implementation * CustomTransformMultiOutputs is in another PR, so remove it * add LDC and LAL convention correction * use pycbc standard names * more meaningful name * use pycbc standard names * Update relbin.py * Update parameters.py * remove unnecessary changes * fix cc issue * fix cc issue * fix cc issue * fix cc issue * compactify * compactify * add __all__ back * Update transforms.py * Update transforms.py * Update test_transforms.py * Update transforms.py * update doc * fix time warning * Update space.py * Update test_transforms.py * Create test_coordinates_space.py * fix cc issues * fix cc issues * fix cc issue * Update tox.ini * Update tox.ini * Update tox.ini * Update tox.ini * Update tox.ini * Update tox.ini * Update tox.ini * Update test_coordinates_space.py * add inline doc * Update tox.ini * add check of bbhx * Update test_coordinates_space.py * Update tox.ini * Update test_coordinates_space.py * add MultibandRelativeTimeDom into hierarchical.py * Update __init__.py * Update hierarchical.py * Update hierarchical.py * Update relbin.py * Update hierarchical.py * Update hierarchical.py * Update relbin.py * Update hierarchical.py * Update hierarchical.py * Update hierarchical.py * Update hierarchical.py * Update hierarchical.py * Update relbin.py * Update hierarchical.py * Update hierarchical.py * Update relbin.py * Update __init__.py * Update space.py * Update space.py * Update space.py * fix psi issue * Update test_coordinates_space.py * Update test_coordinates_space.py * update lalsimulation cvmfs path (#4580) * update lalsimulation cvmfs path * missed that the mount location needs to be changed as well * more references to previsou CVMFS location * Revert "add coordinates_space.py (#4289)" This reverts commit e3418c7. * REmoving lisa examples * REmove inference examples * Remove LISA deps * Removing more LISA things --------- Co-authored-by: Gareth S Cabourn Davies <gareth.cabourndavies@ligo.org> Co-authored-by: Shichao Wu <wushichao@mail.bnu.edu.cn>
gwastro · Jan 9, 2024 · 132e9da · 132e9da
1 parent 519082a
commit 132e9da
Show file tree

Hide file tree

Showing 27 changed files with 306 additions and 247 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -15,7 +15,7 @@ RUN dnf -y install https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-
 
 # set up environment
 RUN cd / && \
-    mkdir -p /cvmfs/config-osg.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org /cvmfs/gwosc.osgstorage.org && echo "config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org cvmfs ro,noauto 0 0" >> /etc/fstab && echo "oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org cvmfs ro,noauto 0 0" >> /etc/fstab && echo "gwosc.osgstorage.org /cvmfs/gwosc.osgstorage.org cvmfs ro,noauto 0 0" >> /etc/fstab && mkdir -p /oasis /scratch /projects /usr/lib64/slurm /var/run/munge && \
+    mkdir -p /cvmfs/config-osg.opensciencegrid.org /cvmfs/software.igwn.org /cvmfs/gwosc.osgstorage.org && echo "config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org cvmfs ro,noauto 0 0" >> /etc/fstab && echo "software.igwn.org /cvmfs/software.igwn.org cvmfs ro,noauto 0 0" >> /etc/fstab && echo "gwosc.osgstorage.org /cvmfs/gwosc.osgstorage.org cvmfs ro,noauto 0 0" >> /etc/fstab && mkdir -p /oasis /scratch /projects /usr/lib64/slurm /var/run/munge && \
     groupadd -g 1000 pycbc && useradd -u 1000 -g 1000 -d /opt/pycbc -k /etc/skel -m -s /bin/bash pycbc
 
 # Install MPI software needed for pycbc_inference
@@ -37,7 +37,7 @@ ENV PATH "/usr/local/bin:/usr/bin:/bin:/lib64/openmpi/bin/bin"
 # Set the default LAL_DATA_PATH to point at CVMFS first, then the container.
 # Users wanting it to point elsewhere should start docker using:
 #   docker <cmd> -e LAL_DATA_PATH="/my/new/path"
-ENV LAL_DATA_PATH "/cvmfs/oasis.opensciencegrid.org/ligo/sw/pycbc/lalsuite-extra/current/share/lalsimulation:/opt/pycbc/pycbc-software/share/lal-data"
+ENV LAL_DATA_PATH "/cvmfs/software.igwn.org/pycbc/lalsuite-extra/current/share/lalsimulation:/opt/pycbc/pycbc-software/share/lal-data"
 
 # When the container is started with
 #   docker run -it pycbc/pycbc-el8:latest

diff --git a/bin/all_sky_search/pycbc_add_statmap b/bin/all_sky_search/pycbc_add_statmap
@@ -345,6 +345,8 @@ else:
                 f_in.attrs['background_time_exc'],
                 **significance_dict[ifo_combo_key])
 
+del ifo_combo_key
+
 logging.info('Combining false alarm rates from all available backgrounds')
 
 # Convert dictionary of whether the ifo combination is available at trigger
@@ -361,8 +363,14 @@ fg_fars_out = np.sum(isincombo_mask * fg_fars, axis=0)
 fg_fars_exc_out = np.sum(isincombo_mask * fg_fars_exc, axis=0)
 
 # Apply any limits as appropriate
-fg_fars_out = significance.apply_far_limit(fg_fars_out, significance_dict, combo=fg_coinc_type)
-fg_fars_exc_out = significance.apply_far_limit(fg_fars_exc_out, significance_dict, combo=fg_coinc_type)
+fg_fars_out = significance.apply_far_limit(
+    fg_fars_out,
+    significance_dict,
+    combo=fg_coinc_type)
+fg_fars_exc_out = significance.apply_far_limit(
+    fg_fars_exc_out,
+    significance_dict,
+    combo=fg_coinc_type)
 
 fg_ifar = conv.sec_to_year(1. / fg_fars_out)
 fg_ifar_exc = conv.sec_to_year(1. / fg_fars_exc_out)
@@ -562,6 +570,7 @@ while True:
     final_combined_fg = final_combined_fg + \
                             combined_fg_data.select(where_combined)
     combined_fg_data = combined_fg_data.remove(where_combined)
+    fg_coinc_type = np.delete(fg_coinc_type, where_combined)
     n_triggers -= 1
 
     logging.info('Removing background triggers at time {} within window '
@@ -604,7 +613,18 @@ while True:
             sep_fg_data[key].data['stat'],
             sep_bg_data[key].data['decimation_factor'],
             bg_t_y,
-            **significance_dict[ifo_combo_key])
+            **significance_dict[key])
+        fg_far = significance.apply_far_limit(
+            fg_far,
+            significance_dict,
+            combo=key,
+        )
+        bg_far = significance.apply_far_limit(
+            bg_far,
+            significance_dict,
+            combo=key,
+        )
+
         sep_bg_data[key].data['ifar'] = 1. / bg_far
         sep_fg_data[key].data['ifar'] = 1. / fg_far
         sep_fg_data[key].data['fap'] = 1 - \
@@ -617,7 +637,7 @@ while True:
             combined_fg_data.data['stat'],
             sep_bg_data[key].data['decimation_factor'],
             bg_time_ct[key],
-            **significance_dict[ifo_combo_key])
+            **significance_dict[key])
         # Set up variable for whether each coincidence is available in each coincidence time
         is_in_combo_time[key] = np.zeros(n_triggers)
         end_times = np.array(f['segments/%s/end' % key][:])
@@ -631,9 +651,15 @@ while True:
     isincombo_mask = np.array([list(is_in_combo_time[ct])
                                for ct in all_ifo_combos])
     fg_fars = np.array([list(far[ct]) for ct in all_ifo_combos])
+    fg_fars_out = np.sum(isincombo_mask * fg_fars, axis=0)
+    fg_fars_out = significance.apply_far_limit(
+        fg_fars_out,
+        significance_dict,
+        combo=fg_coinc_type,
+    )
     # Combine the FARs with the mask to obtain the new ifars
     combined_fg_data.data['ifar'] = conv.sec_to_year(
-        1. / np.sum(isincombo_mask * fg_fars, axis=0))
+        1. / fg_fars_out)
     fg_time -= args.cluster_window
     combined_fg_data.data['fap'] = 1 - \
         np.exp(-conv.sec_to_year(fg_time) / combined_fg_data.data['ifar'])

diff --git a/bin/all_sky_search/pycbc_coinc_statmap b/bin/all_sky_search/pycbc_coinc_statmap
@@ -257,10 +257,22 @@ bg_far_exc, fg_far_exc = significance.get_far(
     background_time_exc,
     **significance_dict[ifo_combo])
 
-fg_far = significance.apply_far_limit(fg_far, significance_dict, combo=ifo_combo)
-bg_far = significance.apply_far_limit(bg_far, significance_dict, combo=ifo_combo)
-fg_far_exc = significance.apply_far_limit(fg_far_exc, significance_dict, combo=ifo_combo)
-bg_far_exc = significance.apply_far_limit(bg_far_exc, significance_dict, combo=ifo_combo)
+fg_far = significance.apply_far_limit(
+    fg_far,
+    significance_dict,
+    combo=ifo_combo)
+bg_far = significance.apply_far_limit(
+    bg_far,
+    significance_dict,
+    combo=ifo_combo)
+fg_far_exc = significance.apply_far_limit(
+    fg_far_exc,
+    significance_dict,
+    combo=ifo_combo)
+bg_far_exc = significance.apply_far_limit(
+    bg_far_exc,
+    significance_dict,
+    combo=ifo_combo)
 
 f['background/ifar'] = conv.sec_to_year(1. / bg_far)
 f['background_exc/ifar'] = conv.sec_to_year(1. / bg_far_exc)
@@ -421,6 +433,17 @@ while numpy.any(ifar_foreground >= background_time):
         return_counts=True,
         **significance_dict[ifo_combo])
 
+    fg_far = significance.apply_far_limit(
+        fg_far,
+        significance_dict,
+        combo=ifo_combo
+    )
+    bg_far = significance.apply_far_limit(
+        bg_far,
+        significance_dict,
+        combo=ifo_combo,
+    )
+
     # Update the ifar_foreground criteria depending on whether foreground
     # triggers are being removed via inclusive or exclusive background.   
     if args.hierarchical_removal_against == 'inclusive':
@@ -435,6 +458,11 @@ while numpy.any(ifar_foreground >= background_time):
             exc_zero_trigs.decimation_factor,
             background_time_exc,
             **significance_dict[ifo_combo])
+        fg_far_exc = significance.apply_far_limit(
+            fg_far_exc,
+            significance_dict,
+            combo=ifo_combo
+        )
         ifar_foreground = 1. / fg_far_exc
     # ifar_foreground has been updated and the code can continue.
 

diff --git a/bin/all_sky_search/pycbc_coinc_statmap_inj b/bin/all_sky_search/pycbc_coinc_statmap_inj
@@ -37,12 +37,21 @@ if args.verbose:
     log_level = logging.INFO
     logging.basicConfig(format='%(asctime)s : %(message)s', level=log_level)
 
-ifo_key = ''.join(args.ifos)
-significance_dict = significance.digest_significance_options([ifo_key], args)
 
 window = args.cluster_window
 logging.info("Loading coinc zerolag triggers")
 zdata = pycbc.io.MultiifoStatmapData(files=args.zero_lag_coincs, ifos=args.ifos)
+
+if 'ifos' in zdata.attrs:
+    ifos = zdata.attrs['ifos'].split(' ')
+    logging.info('using ifos from file {}'.format(args.zero_lag_coincs[0]))
+else:
+    ifos = args.ifos
+    logging.info('using ifos from command line input')
+
+ifo_key = ''.join(ifos)
+significance_dict = significance.digest_significance_options([ifo_key], args)
+
 zdata = zdata.cluster(window)
 
 f = h5py.File(args.output_file, "w")
@@ -51,7 +60,7 @@ f.attrs['num_of_ifos'] = zdata.attrs['num_of_ifos']
 f.attrs['pivot'] = zdata.attrs['pivot']
 f.attrs['fixed'] = zdata.attrs['fixed']
 f.attrs['timeslide_interval'] = zdata.attrs['timeslide_interval']
-f.attrs['ifos'] = ' '.join(sorted(args.ifos))
+f.attrs['ifos'] = ' '.join(sorted(ifos))
 
 # Copy over the segment for coincs and singles
 for key in zdata.seg.keys():
@@ -90,7 +99,11 @@ if len(zdata) > 0:
         background_time,
         **significance_dict[ifo_key])
 
-    fg_far_exc = significance.apply_far_limit(fg_far_exc, significance_dict, combo=ifo_key)
+    fg_far_exc = significance.apply_far_limit(
+        fg_far_exc,
+        significance_dict,
+        combo=ifo_key,
+    )
 
     ifar_exc = 1. / fg_far_exc
     fap_exc = 1 - numpy.exp(- coinc_time / ifar_exc)

diff --git a/bin/all_sky_search/pycbc_sngls_statmap b/bin/all_sky_search/pycbc_sngls_statmap
@@ -146,8 +146,16 @@ bg_far, fg_far = significance.get_far(
     fg_time,
     **significance_dict[ifo])
 
-fg_far = significance.apply_far_limit(fg_far, significance_dict, combo=ifo)
-bg_far = significance.apply_far_limit(bg_far, significance_dict, combo=ifo)
+fg_far = significance.apply_far_limit(
+    fg_far,
+    significance_dict,
+    combo=ifo,
+)
+bg_far = significance.apply_far_limit(
+    bg_far,
+    significance_dict,
+    combo=ifo,
+)
 
 bg_ifar = 1. / bg_far
 fg_ifar = 1. / fg_far
@@ -341,6 +349,18 @@ while numpy.any(ifar_louder > hier_ifar_thresh_s):
         fg_time,
         **significance_dict[ifo])
 
+    fg_far = significance.apply_far_limit(
+        fg_far,
+        significance_dict,
+        combo=ifo,
+    )
+
+    bg_far = significance.apply_far_limit(
+        bg_far,
+        significance_dict,
+        combo=ifo,
+    )
+
     bg_ifar = 1. / bg_far
     fg_ifar = 1. / fg_far
 
@@ -359,6 +379,12 @@ while numpy.any(ifar_louder > hier_ifar_thresh_s):
             fg_time_exc,
             **significance_dict[ifo])
 
+        fg_far_exc = significance.apply_far_limit(
+            fg_far_exc,
+            significance_dict,
+            combo=ifo,
+        )
+
         fg_ifar_exc = 1. / fg_far_exc
 
         ifar_louder = fg_ifar_exc

diff --git a/bin/all_sky_search/pycbc_sngls_statmap_inj b/bin/all_sky_search/pycbc_sngls_statmap_inj
@@ -117,8 +117,14 @@ bg_far_exc, fg_far_exc = significance.get_far(
     fg_time_exc,
     **significance_dict[ifo])
 
-fg_far_exc = significance.apply_far_limit(fg_far_exc, significance_dict, combo=ifo)
-bg_far_exc = significance.apply_far_limit(bg_far_exc, significance_dict, combo=ifo)
+fg_far_exc = significance.apply_far_limit(
+    fg_far_exc,
+    significance_dict,
+    combo=ifo)
+bg_far_exc = significance.apply_far_limit(
+    bg_far_exc,
+    significance_dict,
+    combo=ifo)
 
 fg_ifar_exc = 1. / fg_far_exc
 bg_ifar_exc = 1. / bg_far_exc

diff --git a/bin/minifollowups/pycbc_page_snglinfo b/bin/minifollowups/pycbc_page_snglinfo
@@ -138,8 +138,11 @@ else:
 
 if args.ranking_statistic in ["quadsum", "single_ranking_only"]:
     stat_name = sngl_stat_name
+    stat_name_long = sngl_stat_name
 else:
-    stat_name = '_with_'.join(ranking_statistic, sngl_ranking)
+    # Name would be too long - just call it ranking statistic
+    stat_name = 'Ranking Statistic'
+    stat_name_long = ' with '.join([args.ranking_statistic, args.sngl_ranking])
 
 headers.append(stat_name)
 
@@ -201,7 +204,7 @@ html = pycbc.results.dq.redirect_javascript + \
 if args.n_loudest:
     title = 'Parameters of single-detector event ranked %s' \
         % (args.n_loudest + 1)
-    caption = 'Parameters of the single-detector event ranked number %s by %s. The figures below show the mini-followup data for this event.' % (args.n_loudest + 1, stat_name)
+    caption = 'Parameters of the single-detector event ranked number %s by %s. The figures below show the mini-followup data for this event.' % (args.n_loudest + 1, stat_name_long)
 else:
     title = 'Parameters of single-detector event'
     caption = 'Parameters of the single-detector event. The figures below show the mini-followup data for this event.'

diff --git a/bin/minifollowups/pycbc_plot_trigger_timeseries b/bin/minifollowups/pycbc_plot_trigger_timeseries
@@ -92,21 +92,11 @@ for ifo in args.single_trigger_files.keys():
     logging.info("Getting %s", args.plot_type)
     rank = ranking.get_sngls_ranking_from_trigs(trigs, args.plot_type)
 
-    if all(rank == 1):
-        # This is the default value to say "downranked beyond consideration"
-        # so skip these events
-        pylab.scatter(-2 * args.window, 0,
-                      color=pycbc.results.ifo_color(ifo),
-                      marker='x',
-                      label=ifo)
-        continue
-
     pylab.scatter(trigs['end_time'] - t, rank,
                   color=pycbc.results.ifo_color(ifo), marker='x',
                   label=ifo)
 
-    # Minimum rank which hasn't been set to the default of 1
-    min_rank = min(min_rank, rank[rank > 1].min())
+    min_rank = min(min_rank, rank.min())
 
     if args.special_trigger_ids:
         special_idx = args.special_trigger_ids[ifo]
@@ -138,7 +128,10 @@ logging.info("Saving figure")
 pycbc.results.save_fig_with_metadata(fig, args.output_file,
             cmd = ' '.join(sys.argv),
             title = 'Single Detector Trigger Timeseries (%s)' % args.plot_type,
-            caption = 'Time series showing the single detector triggers'
-                      ' centered around the time of the trigger of interest.',
+            caption = 'Time series showing the single-detector triggers '
+                      'centered around the time of the trigger of interest. '
+                      'Triggers with ranking 1 have been downweighted beyond '
+                      'consideration, but may still form insignificant '
+                      'events.',
          )
 logging.info("Done!")