Merge pull request #102 from lsst/tickets/PREOPS-4102

tickets/PREOPS-4102: Support customizing which columns are shown in schedview's Times Square reports

docs/playwright_testing.rst

@@ -73,4 +73,3 @@ Run test generator from another terminal:
 ::
 
  playwright codegen http://localhost:8080/schedview-snapshot/dashboard
-

pyproject.toml

@@ -90,6 +90,7 @@ line_length = 110
 [tool.ruff]
 exclude = [
     "__init__.py",
+    "notebooks",
 ]
 ignore = [
     "N802",

schedview/collect/opsim.py

@@ -4,68 +4,29 @@
 import yaml
 from astropy.time import Time
 from lsst.resources import ResourcePath
+from rubin_scheduler.scheduler.utils import SchemaConverter
 from rubin_scheduler.utils import ddf_locations
 
 try:
     from rubin_sim import maf
 except ModuleNotFoundError:
     pass
 
-DEFAULT_VISITS_COLUMNS = [
-    "observationId",
-    "fieldRA",
-    "fieldDec",
-    "observationStartMJD",
-    "flush_by_mjd",
-    "visitExposureTime",
-    "filter",
-    "rotSkyPos",
-    "rotSkyPos_desired",
-    "numExposures",
-    "airmass",
-    "seeingFwhm500",
-    "seeingFwhmEff",
-    "seeingFwhmGeom",
-    "skyBrightness",
-    "night",
-    "slewTime",
-    "visitTime",
-    "slewDistance",
-    "fiveSigmaDepth",
-    "altitude",
-    "azimuth",
-    "paraAngle",
-    "cloud",
-    "moonAlt",
-    "sunAlt",
-    "note",
-    "target",
-    "fieldId",
-    "proposalId",
-    "block_id",
-    "observationStartLST",
-    "rotTelPos",
-    "rotTelPos_backup",
-    "moonAz",
-    "sunAz",
-    "sunRA",
-    "sunDec",
-    "moonRA",
-    "moonDec",
-    "moonDistance",
-    "solarElong",
-    "moonPhase",
-    "cummTelAz",
-    "scripted_id",
-]
+
+def _all_visits_columns():
+    """Return all visits columns understood by the current rubin_scheduler."""
+    schema_converter = SchemaConverter()
+    current_cols = set(schema_converter.convert_dict.keys())
+    backwards_cols = set(schema_converter.backwards.keys())
+    return current_cols.union(backwards_cols)
 
 
 def read_opsim(
     opsim_uri,
     start_time=None,
     end_time=None,
     constraint=None,
-    dbcols=DEFAULT_VISITS_COLUMNS,
+    dbcols=None,
     **kwargs,
 ):
     """Read visits from an opsim database.
@@ -80,8 +41,9 @@ def read_opsim(
         The end time for visits ot be loaded
     constraint : `str`, None
         Query for which visits to load.
-    dbcols : `list` [`str`]
-        Columns required from the database.
+    dbcols : `None` or `list` [`str`]
+        Columns required from the database. Defaults to None, which queries
+        all columns known to rubin_scheduler.
     **kwargs
         Passed to `maf.get_sim_data`, if `rubin_sim` is available.
 
@@ -121,6 +83,12 @@ def read_opsim(
 
     with obs_path.as_local() as local_obs_path:
         with sqlite3.connect(local_obs_path.ospath) as sim_connection:
+            if dbcols is None:
+                col_query = "SELECT name FROM PRAGMA_TABLE_INFO('observations')"
+                dbcols = [
+                    c for c in pd.read_sql(col_query, sim_connection).name if c in _all_visits_columns()
+                ]
+
             try:
                 visits = pd.DataFrame(maf.get_sim_data(sim_connection, constraint, dbcols, **kwargs))
             except NameError as e:
@@ -147,6 +115,7 @@ def read_opsim(
                         visits.observationStartMJD + 2400000.5, origin="julian", unit="D", utc=True
                     )
 
+    visits.rename(columns=SchemaConverter().backwards, inplace=True)
     visits.set_index("observationId", inplace=True)
 
     return visits
@@ -156,7 +125,7 @@ def read_ddf_visits(
     opsim_uri,
     start_time=None,
     end_time=None,
-    dbcols=DEFAULT_VISITS_COLUMNS,
+    dbcols=None,
     **kwargs,
 ):
     """Read DDF visits from an opsim database.
@@ -169,8 +138,9 @@ def read_ddf_visits(
         The start time for visits to be loaded
     end_time : `str`, `astropy.time.Time`
         The end time for visits ot be loaded
-    dbcols : `list` [`str`]
-        Columns required from the database.
+    dbcols : `None` or `list` [`str`]
+        Columns required from the database. Defaults to None, which queries
+        all columns known to rubin_scheduler.
     stackers : `list` [`rubin_sim.maf.stackers`], optional
         Stackers to be used to generate additional columns.
 
@@ -179,6 +149,7 @@ def read_ddf_visits(
     visits : `pandas.DataFrame`
         The visits and their parameters.
     """
+
     ddf_field_names = tuple(ddf_locations().keys())
     constraint = f"target IN {tuple(field_name for field_name in ddf_field_names)}"
     visits = read_opsim(

schedview/plot/__init__.py

@@ -29,6 +29,7 @@
     "PLOT_FILTER_COLORS",
     "PLOT_FILTER_CMAP",
     "create_cadence_plot",
+    "create_visit_table",
 ]
 
 from .cadence import create_cadence_plot
@@ -52,4 +53,4 @@
 )
 from .survey import create_hpix_visit_map_grid, map_survey_healpix, map_visits_over_hpix
 from .visitmap import create_visit_skymaps, plot_visit_planisphere, plot_visit_skymaps
-from .visits import create_visit_explorer, plot_visit_param_vs_time, plot_visits
+from .visits import create_visit_explorer, create_visit_table, plot_visit_param_vs_time, plot_visits

schedview/plot/nightly.py

@@ -6,6 +6,7 @@
 import numpy as np
 
 from .visitmap import BAND_COLORS
+from .visits import visits_tooltips
 
 DEFAULT_EVENT_LABELS = {
     "sunset": "sunset",
@@ -41,78 +42,6 @@
 }
 
 
-def _visits_tooltips(visits, weather=False):
-    deg = "\u00b0"
-    tooltips = [
-        (
-            "Start time",
-            "@start_date{%F %T} UTC (mjd=@observationStartMJD{00000.00}, LST=@observationStartLST"
-            + deg
-            + ")",
-        ),
-        ("flush by mjd", "@flush_by_mjd{00000.00}"),
-        ("Note", "@note"),
-        ("Filter", "@filter"),
-        (
-            "Field coordinates",
-            "RA=@fieldRA" + deg + ", Decl=@fieldDec" + deg + ", Az=@azimuth" + deg + ", Alt=@altitude" + deg,
-        ),
-        ("Parallactic angle", "@paraAngle" + deg),
-        ("Rotator angle", "@rotTelPos" + deg),
-        ("Rotator angle (backup)", "@rotTelPos_backup" + deg),
-        ("Cumulative telescope azimuth", "@cummTelAz" + deg),
-        ("Airmass", "@airmass"),
-        ("Moon distance", "@moonDistance" + deg),
-        (
-            "Moon",
-            "RA=@sunRA"
-            + deg
-            + ", Decl=@sunDec"
-            + deg
-            + ", Az=@sunAz"
-            + deg
-            + ", Alt=@sunAlt"
-            + deg
-            + ", phase=@moonPhase"
-            + deg,
-        ),
-        (
-            "Sun",
-            "RA=@moonRA"
-            + deg
-            + ", Decl=@moonDec"
-            + deg
-            + ", Az=@moonAz"
-            + deg
-            + ", Alt=@moonAlt"
-            + deg
-            + ", elong=@solarElong"
-            + deg,
-        ),
-        ("Sky brightness", "@skyBrightness mag arcsec^-2"),
-        ("Exposure time", "@visitExposureTime seconds (@numExposures exposures)"),
-        ("Visit time", "@visitTime seconds"),
-        ("Slew distance", "@slewDistance" + deg),
-        ("Slew time", "@slewTime seconds"),
-        ("Field ID", "@fieldId"),
-        ("Proposal ID", "@proposalId"),
-        ("Block ID", "@block_id"),
-        ("Scripted ID", "@scripted_id"),
-    ]
-
-    if weather:
-        tooltips += [
-            (
-                "Seeing",
-                '@seeingFwhm500" (500nm), @seeingFwhmEff" (Eff), @seeingFwhmGeom" (Geom)',
-            ),
-            ("Cloud", "@cloud"),
-            ("5-sigma depth", "@fiveSigmaDepth"),
-        ]
-
-    return tooltips
-
-
 def _add_almanac_events(fig, almanac_events, event_labels, event_colors):
     for event, row in almanac_events.iterrows():
         if event_labels[event] is None:
@@ -198,7 +127,7 @@ def plot_airmass_vs_time(
 
     hover_tool = bokeh.models.HoverTool()
     hover_tool.renderers = fig.select({"name": "visit_airmass"})
-    hover_tool.tooltips = _visits_tooltips(visits)
+    hover_tool.tooltips = visits_tooltips()
     hover_tool.formatters = {"@start_date": "datetime"}
     fig.add_tools(hover_tool)
 
@@ -327,7 +256,7 @@ def plot_alt_vs_time(
 
     hover_tool = bokeh.models.HoverTool()
     hover_tool.renderers = fig.select({"name": "visit_altitude"})
-    hover_tool.tooltips = _visits_tooltips(visits)
+    hover_tool.tooltips = visits_tooltips()
     hover_tool.formatters = {"@start_date": "datetime"}
     fig.add_tools(hover_tool)
 
@@ -539,7 +468,7 @@ def plot_polar_alt_az(visits, band_shapes=BAND_SHAPES, figure=None, legend=True)
 
     hover_tool = bokeh.models.HoverTool()
     hover_tool.renderers = fig.select({"name": "visit_altitude"})
-    hover_tool.tooltips = _visits_tooltips(visits)
+    hover_tool.tooltips = visits_tooltips()
     hover_tool.formatters = {"@start_date": "datetime"}
     fig.add_tools(hover_tool)