tickets/PREOPS-4948: functions to support imitating a DES nightsum report using a Times Square notebook

.github/workflows/cache.yaml

@@ -29,10 +29,10 @@ jobs:
         id: rs-install
         shell: bash -l {0}
         run: |
-          mamba install --quiet rubin-scheduler     
+          mamba install --quiet rubin-scheduler
           mamba list rubin-scheduler | grep -v "#" | awk '{print $2}' > ${{ github.workspace }}/rs_version
           echo "rs-version" `cat ${{ github.workspace }}/rs_version`
-          echo "rs-version=`cat ${{ github.workspace }}/rs_version`" >> $GITHUB_OUTPUT          
+          echo "rs-version=`cat ${{ github.workspace }}/rs_version`" >> $GITHUB_OUTPUT
 
       - name: Access rubin-sched-data cache
         id: cache-rs
@@ -59,7 +59,7 @@ jobs:
         name: Update/Download data.
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data       
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
           scheduler_download_data --update --tdqm_disable
 
       - name: Check data
@@ -72,6 +72,3 @@ jobs:
           cat $RUBIN_SIM_DATA_DIR/versions.txt
           echo "is it in sync?"
           scheduler_download_data --versions
-
-
-

.github/workflows/cleanup_cache.yaml

@@ -15,7 +15,7 @@ jobs:
           echo "Fetching list of cache key"
           cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH  | cut -f 1)
 
-          ## Setting this to not fail the workflow while deleting cache keys. 
+          ## Setting this to not fail the workflow while deleting cache keys.
           set +e
           echo "Deleting caches..."
           for cacheKey in $cacheKeysForPR
@@ -27,6 +27,3 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
           BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge
-
-
-

.github/workflows/ruff.yaml

@@ -21,4 +21,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: chartboost/ruff-action@v1
+      - uses: chartboost/ruff-action@v1

.gitignore

@@ -146,3 +146,7 @@ util/sample_data/sample_rewards.h5
 util/sample_data/sample_scheduler.pickle.xz
 notebooks/de421.bsp
 notebooks/MPCORB.DAT.gz
+
+# Other temporary files
+tmp/*
+util/tmp/*

container_environment.yaml

@@ -9,6 +9,7 @@ dependencies:
  - pandas
  - astropy >= 5.3
  - rubin-scheduler
+ - rubin-sim
  - uranography >= 1.1.0
  - bokeh >= 3.1.1
  - panel >= 1.1.0

requirements.txt

@@ -11,4 +11,5 @@ param
 pytz
 uranography
 rubin-scheduler
+rubin-sim
 pip

schedview/app/prenight/prenight.py

@@ -287,12 +287,20 @@ def _update_visits(self):
             if not ResourcePath(self.opsim_output_fname).exists():
                 raise FileNotFoundError(f"Resource not found: {self.opsim_output_fname}")
 
-            visits = schedview.collect.opsim.read_opsim(
-                self.opsim_output_fname,
-                Time(self._almanac_events.loc["sunset", "UTC"]),
-                Time(self._almanac_events.loc["sunrise", "UTC"]),
-            )
-            if len(visits) == 0:
+            try:
+                visits = schedview.collect.opsim.read_opsim(
+                    self.opsim_output_fname,
+                    Time(self._almanac_events.loc["sunset", "UTC"]),
+                    Time(self._almanac_events.loc["sunrise", "UTC"]),
+                )
+                no_data_found = len(visits) == 0
+            except UserWarning as user_warning:
+                if user_warning.args[0].startswith("No data found matching"):
+                    no_data_found = True
+                else:
+                    raise user_warning
+
+            if no_data_found:
                 self.logger.info("No visits on requested night, looking for a night with visits.")
                 # read all visits to so we can find a central one
                 visits = schedview.collect.opsim.read_opsim(self.opsim_output_fname)

schedview/collect/__init__.py

@@ -4,6 +4,7 @@
     "read_scheduler",
     "sample_pickle",
     "load_bright_stars",
+    "read_ddf_visits",
 ]
 
 from .footprint import get_footprint

schedview/collect/local.py

@@ -1,5 +1,5 @@
 from .metrics import get_metric_path  # noqa F401
-from .opsim import read_opsim  # noqa F401
+from .opsim import read_ddf_visits, read_opsim  # noqa F401
 from .scheduler_pickle import read_scheduler  # noqa F401
 from .scheduler_pickle import sample_pickle  # noqa F401
 from .stars import load_bright_stars  # noqa F401

schedview/collect/opsim.py

@@ -1,13 +1,99 @@
 import sqlite3
 
-import numpy as np
 import pandas as pd
 import yaml
 from astropy.time import Time
 from lsst.resources import ResourcePath
+from rubin_scheduler.utils import ddf_locations
+from rubin_sim import maf
 
+DEFAULT_VISITS_COLUMNS = [
+    "observationId",
+    "fieldRA",
+    "fieldDec",
+    "observationStartMJD",
+    "flush_by_mjd",
+    "visitExposureTime",
+    "filter",
+    "rotSkyPos",
+    "rotSkyPos_desired",
+    "numExposures",
+    "airmass",
+    "seeingFwhm500",
+    "seeingFwhmEff",
+    "seeingFwhmGeom",
+    "skyBrightness",
+    "night",
+    "slewTime",
+    "visitTime",
+    "slewDistance",
+    "fiveSigmaDepth",
+    "altitude",
+    "azimuth",
+    "paraAngle",
+    "cloud",
+    "moonAlt",
+    "sunAlt",
+    "note",
+    "target",
+    "fieldId",
+    "proposalId",
+    "block_id",
+    "observationStartLST",
+    "rotTelPos",
+    "rotTelPos_backup",
+    "moonAz",
+    "sunAz",
+    "sunRA",
+    "sunDec",
+    "moonRA",
+    "moonDec",
+    "moonDistance",
+    "solarElong",
+    "moonPhase",
+    "cummTelAz",
+    "scripted_id",
+]
 
-def read_opsim(opsim_uri, start_time="2000-01-01", end_time="2100-01-01"):
+
+class StartDateStacker(maf.BaseStacker):
+    """Add the start date."""
+
+    cols_added = ["start_date"]
+
+    def __init__(self, start_mjd_col="observationStartMJD"):
+        self.units = "ns"
+        self.cols_req = [start_mjd_col]
+        self.start_mjd_col = start_mjd_col
+
+    def _run(self, sim_data, cols_present=False):
+        """The start date as a datetime."""
+        if cols_present:
+            # Column already present in data; assume it is correct and does not
+            # need recalculating.
+            return sim_data
+        if len(sim_data) == 0:
+            return sim_data
+
+        sim_data["start_date"] = pd.to_datetime(
+            sim_data[self.start_mjd_col] + 2400000.5, origin="julian", unit="D", utc=True
+        )
+
+        return sim_data
+
+
+DEFAULT_STACKERS = [maf.HourAngleStacker(), StartDateStacker()]
+
+
+def read_opsim(
+    opsim_uri,
+    start_time=None,
+    end_time=None,
+    constraint=None,
+    dbcols=DEFAULT_VISITS_COLUMNS,
+    stackers=DEFAULT_STACKERS,
+    **kwargs,
+):
     """Read visits from an opsim database.
 
     Parameters
@@ -18,14 +104,36 @@ def read_opsim(opsim_uri, start_time="2000-01-01", end_time="2100-01-01"):
         The start time for visits to be loaded
     end_time : `str`, `astropy.time.Time`
         The end time for visits ot be loaded
+    constraint : `str`, None
+        Query for which visits to load.
+    dbcols : `list` [`str`]
+        Columns required from the database.
+    stackers : `list` [`rubin_sim.maf.stackers`], optional
+        Stackers to be used to generate additional columns.
 
     Returns
     -------
     visits : `pandas.DataFrame`
         The visits and their parameters.
     """
-    start_mjd = Time(start_time).mjd
-    end_mjd = Time(end_time).mjd
+
+    # Add constraints corresponding to quested start and end times
+    if (start_time is not None) or (end_time is not None):
+        if constraint is None:
+            constraint = ""
+
+        if start_time is not None:
+            if len(constraint) > 0:
+                constraint += " AND "
+            constraint += f"(observationStartMJD >= {Time(start_time).mjd})"
+
+        if end_time is not None:
+            if len(constraint) > 0:
+                constraint += " AND "
+            constraint += f"(observationStartMJD <= {Time(end_time).mjd})"
+
+    if stackers is not None and len(stackers) > 0:
+        kwargs["stackers"] = stackers
 
     original_resource_path = ResourcePath(opsim_uri)
 
@@ -42,18 +150,53 @@ def read_opsim(opsim_uri, start_time="2000-01-01", end_time="2100-01-01"):
 
     with obs_path.as_local() as local_obs_path:
         with sqlite3.connect(local_obs_path.ospath) as sim_connection:
-            visits = pd.read_sql_query(
-                f"SELECT * FROM observations WHERE observationStartMJD BETWEEN {start_mjd} AND {end_mjd}",
-                sim_connection,
-                index_col="observationId",
-            )
-
-    visits["start_date"] = pd.to_datetime(
-        visits["observationStartMJD"] + 2400000.5, origin="julian", unit="D", utc=True
-    )
+            visits = pd.DataFrame(maf.get_sim_data(sim_connection, constraint, dbcols, **kwargs))
+
+    if "start_date" in visits:
+        visits["start_date"] = pd.to_datetime(visits.start_date, unit="ns", utc=True)
 
-    if "HA_hours" not in visits.columns:
-        visits["HA_hours"] = (visits.observationStartLST - visits.fieldRA) * 24.0 / 360.0
-        visits["HA_hours"] = np.mod(visits["HA_hours"] + 12.0, 24) - 12
+    visits.set_index("observationId", inplace=True)
+
+    return visits
 
+
+def read_ddf_visits(
+    opsim_uri,
+    start_time=None,
+    end_time=None,
+    dbcols=DEFAULT_VISITS_COLUMNS,
+    stackers=DEFAULT_STACKERS,
+    **kwargs,
+):
+    """Read DDF visits from an opsim database.
+
+    Parameters
+    ----------
+    opsim_uri : `str`
+        The uri from which to load visits
+    start_time : `str`, `astropy.time.Time`
+        The start time for visits to be loaded
+    end_time : `str`, `astropy.time.Time`
+        The end time for visits ot be loaded
+    dbcols : `list` [`str`]
+        Columns required from the database.
+    stackers : `list` [`rubin_sim.maf.stackers`], optional
+        Stackers to be used to generate additional columns.
+
+    Returns
+    -------
+    visits : `pandas.DataFrame`
+        The visits and their parameters.
+    """
+    ddf_field_names = tuple(ddf_locations().keys())
+    constraint = f"target IN {tuple(field_name for field_name in ddf_field_names)}"
+    visits = read_opsim(
+        opsim_uri,
+        start_time=start_time,
+        end_time=end_time,
+        constraint=constraint,
+        dbcols=dbcols,
+        stackers=stackers,
+        **kwargs,
+    )
     return visits

schedview/compute/__init__.py

@@ -10,10 +10,14 @@
     "make_scheduler_summary_df",
     "make_survey_reward_df",
     "compute_maps",
+    "compute_metric_by_visit",
+    "compute_hpix_metric_in_bands",
+    "visits",
 ]
 
 from .astro import convert_evening_date_to_night_of_survey, night_events
 from .camera import LsstCameraFootprintPerimeter
+from .maf import compute_hpix_metric_in_bands, compute_metric_by_visit
 from .scheduler import (
     compute_basis_function_reward_at_time,
     compute_basis_function_rewards,