Docker build script, updates to maf trackingDb, container workflow

.github/workflows/build_container.yaml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: lsst-sqre/build-and-push-to-ghcr@v1
+      - uses: lsst-sqre/build-and-push-to-ghcr@tickets/DM-41857
         id: build
         with:
           image: ${{ github.repository }} 

Dockerfile

@@ -3,20 +3,27 @@
 # Base container
 FROM mambaorg/micromamba:1.5.9
 
-# Add rubin-sim from conda-forge
+# Copy current directory
+COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/${MAMBA_USER}/rubin_sim
 
-RUN micromamba install -y -n base -c conda-forge rubin-sim
-RUN micromamba install -y -n base -c conda-forge \
-       jinja2 \
-       tornado
+# Install container requirements from conda-forge
+# Note that open-orb dependencies are omitted
+RUN micromamba install -y -n base -f /home/${MAMBA_USER}/rubin_sim/container_environment.yaml
 RUN micromamba clean --all --yes
+
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
 
+# Install current version of rubin-sim
+RUN python -m pip install /home/$MAMBA_USER/rubin_sim --no-deps
+
 # Container execution
 # Mount fbs simulation outputs expected at /data/fbs_sims
+# Mount rubin_sim_data (if needed) at /data/rubin_sim_data
+
+EXPOSE 80
+ENV PORT=80
 
-EXPOSE 8080
-ENV PORT 8080
+ENV RUBIN_SIM_DATA_DIR=/data/rubin_sim_data
 
-# Start up show_maf on port 8080
+# Start up show_maf on port 80
 CMD cd /data/fbs_sims && show_maf -p 80 --no_browser

container_environment.yaml

@@ -0,0 +1,28 @@
+name: base
+channels:
+ - conda-forge
+dependencies:
+ - numpy <2
+ - matplotlib-base
+ - healpy
+ - pandas
+ - pyarrow
+ - numexpr
+ - scipy
+ - sqlalchemy
+ - astropy
+ - pytables
+ - h5py
+ - astroplan
+ - git
+ - colorcet
+ - cycler
+ - george
+ - scikit-learn
+ - shapely
+ - skyproj
+ - tqdm
+ - jinja2
+ - tornado
+ - rubin-scheduler
+

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
     "skyfield",
     "skyproj",
     "pyoorb",
+    "tqdm",
     "rubin-scheduler",
     "pyarrow",
 ]
@@ -70,6 +71,7 @@ run_moving_fractions = "rubin_sim.maf.run_moving_fractions:run_moving_fractions"
 run_moving_join = "rubin_sim.maf.run_moving_join:run_moving_join"
 scimaf_dir = "rubin_sim.maf.scimaf_dir:scimaf_dir"
 run_selfcal_metric = "rubin_sim.maf.run_selfcal_metric:run_selfcal_metric"
+make_fbs_tracking_db = "rubin_sim.maf.make_fbs_tracking_db:make_fbs_tracking_db"
 show_maf = "rubin_sim.maf.show_maf:show_maf"
 make_lsst_obs = "rubin_sim.moving_objects.make_lsst_obs:make_lsst_obs"
 

requirements.txt

@@ -1,6 +1,6 @@
 setuptools_scm
 setuptools_scm_git_archive
-numpy<2
+numpy
 matplotlib
 healpy
 pandas
@@ -21,5 +21,5 @@ scikit-learn
 shapely
 skyfield
 skyproj
-tdqm
+tqdm
 rubin-scheduler

rubin_sim/maf/db/tracking_db.py

@@ -70,9 +70,11 @@ def __init__(self, database=None, trackingDbverbose=False):
         # connecting to non-existent database creates it automatically
         if database is None:
             # Default is a file in the current directory.
-            self.database = os.path.join(os.getcwd(), "trackingDb_sqlite.db")
+            self.database = "trackingDb_sqlite.db"
+            self.tracking_db_dir = "."
         else:
             self.database = database
+            self.tracking_db_dir = os.path.dirname(database)
         # only sqlite
         dbAddress = url.URL.create(drivername=self.driver, database=self.database)
         engine = create_engine(dbAddress, echo=self.verbose)
@@ -127,6 +129,7 @@ def add_run(
             Set the date the MAF analysis was run.
         maf_dir : `str`, optional
             The relative path to the MAF directory.
+            Will be converted to a relative path if absolute.
         db_file : `str`, optional
             The relative path to the Opsim SQLite database file.
         maf_run_id : `int`, optional
@@ -158,6 +161,8 @@ def add_run(
             maf_date = "NULL"
         if maf_dir is None:
             maf_dir = "NULL"
+        if maf_dir is not None:
+            maf_dir = os.path.relpath(maf_dir, start=self.tracking_db_dir)
         if db_file is None:
             db_file = "NULL"
         # Test if maf_dir already exists in database.
@@ -275,12 +280,12 @@ def add_run_to_database(
     db_file : `str`, optional
         Relative path + name of the opsim database file.
     """
-    maf_dir = os.path.abspath(maf_dir)
+    trackingDb = TrackingDb(database=tracking_db_file)
+
+    maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(trackingDb.tracking_db_dir))
     if not os.path.isdir(maf_dir):
         raise ValueError("There is no directory containing MAF outputs at %s." % (maf_dir))
 
-    trackingDb = TrackingDb(database=tracking_db_file)
-
     # Connect to resultsDb for additional information if available
     if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")) and not skip_extras:
         resdb = ResultsDb(maf_dir)

rubin_sim/maf/make_fbs_tracking_db.py

@@ -0,0 +1,90 @@
+__all__ = ("make_fbs_tracking_db",)
+
+import argparse
+import os
+import sqlite3
+
+import pandas as pd
+
+from rubin_sim.maf.db import ResultsDb, TrackingDb, VersionRow
+
+
+def make_fbs_tracking_db():
+    """Make a database to track lots of opsim outputs,
+    adding in comments and identifiers based on the metric subsets.
+    """
+    parser = argparse.ArgumentParser(
+        description="Create a tracking database for many opsim maf outputs,"
+        "with comments and dates. "
+        "Assumes metrics are in a subdirectory called 'maf'."
+    )
+    parser.add_argument(
+        "--opsim_list",
+        type=str,
+        default="opsim_list",
+        help="File containing a list of all of the opsim runs to add.",
+    )
+    args = parser.parse_args()
+
+    batches = {
+        "meta": "General Info Metrics",
+        "glance": "Quick Look Metrics",
+        "sci": "Science Metrics",
+        "ss": "Solar System Metrics",
+        "ddf": "Deep Drilling Metrics",
+    }
+
+    tracking_db = TrackingDb()
+    print(f"Tracking database in directory {tracking_db.tracking_db_dir}")
+
+    with open(args.opsim_list, "r") as runlist:
+        for run in runlist:
+            db_file = run.replace("\n", "")
+            vals = db_file.split("/")
+            family = vals[-2]
+            db_name = vals[-1]
+            run_name = db_name.replace(".db", "")
+            run_version = run_name.split("_10yrs")[0][-4:]
+            run_group = run_version + family
+
+            # Try to build a comment on the run based on the run_name
+            run_comment = run_name.replace("_10yrs", "")[0:-4]
+            run_comment = run_comment.replace("_", "")
+
+            print(run_name, db_file)
+            conn = sqlite3.connect(db_file)
+            query = "select Value from info where Parameter == 'Date, ymd'"
+            result = pd.read_sql(query, conn)
+            sched_date = result.iloc[0, 0]
+            query = "select Value from info where Parameter like 'rubin_%.__version__'"
+            result = pd.read_sql(query, conn)
+            sched_version = result.iloc[0, 0]
+
+            # Look for metrics in any of the above sets
+            for k in batches:
+                maf_dir = os.path.join("maf", run_name + "_" + k)
+                maf_comment = batches[k]
+                # Get maf run date and version
+                if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")):
+                    resdb = ResultsDb(maf_dir)
+                    resdb.open()
+                    query = resdb.session.query(VersionRow).all()
+                    for v in query:
+                        maf_version = v.version
+                        maf_date = v.run_date
+                    resdb.close()
+
+                    maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(tracking_db.tracking_db_dir))
+                    runId = tracking_db.add_run(
+                        run_group=run_group,
+                        run_name=run_name,
+                        run_comment=run_comment,
+                        run_version=sched_version,
+                        run_date=sched_date,
+                        maf_comment=maf_comment,
+                        maf_version=maf_version,
+                        maf_date=maf_date,
+                        maf_dir=maf_dir,
+                        db_file=db_file,
+                    )
+                    print("Used MAF RunID %d" % (runId))

rubin_sim/maf/utils/sn_n_sn_utils.py

@@ -16,6 +16,7 @@
 from astropy.table import Table
 from rubin_scheduler.data import get_data_dir
 from scipy.interpolate import RegularGridInterpolator
+
 from rubin_sim.maf.utils import m52snr
 
 warnings.filterwarnings("ignore", category=RuntimeWarning)

rubin_sim/maf/web/maf_tracking.py

@@ -25,8 +25,9 @@ class MafTracking:
     def __init__(self, database=None):
         if database is None:
             database = os.path.join(os.getcwd(), "trackingDb_sqlite.db")
+        self.tracking_db = database
 
-        # Read in the results database.
+        # Read in the tracking database.
         cols = [
             "maf_run_id",
             "run_name",
@@ -40,7 +41,7 @@ def __init__(self, database=None):
             "maf_version",
             "maf_date",
         ]
-        self.runs = get_sim_data(database, "", cols, table_name="runs")
+        self.runs = get_sim_data(self.tracking_db, "", cols, table_name="runs")
         self.runs = self.sort_runs(self.runs, order=["maf_run_id", "run_name", "maf_comment"])
         self.runs_page = {}
 
@@ -59,16 +60,17 @@ def run_info(self, run):
             Ordered dict version of the numpy structured array.
         """
         runInfo = OrderedDict()
+        maf_dir = os.path.relpath(run["maf_dir"], start=os.path.dirname(self.tracking_db))
         runInfo["Run Name"] = run["run_name"]
         runInfo["Group"] = run["run_group"]
         runInfo["Maf Comment"] = run["maf_comment"]
         runInfo["Run Comment"] = run["run_comment"]
-        runInfo["SQLite File"] = [
+        runInfo["RunDb File"] = [
             os.path.relpath(run["db_file"]),
             os.path.split(run["db_file"])[1],
         ]
-        runInfo["ResultsDb"] = os.path.relpath(os.path.join(run["maf_dir"], "resultsDb_sqlite.db"))
-        runInfo["maf_dir"] = run["maf_dir"]
+        runInfo["ResultsDb"] = os.path.join(maf_dir, "resultsDb_sqlite.db")
+        runInfo["maf_dir"] = maf_dir
         runInfo["sched_version"] = run["run_version"]
         runInfo["sched_date"] = run["run_date"]
         runInfo["maf_version"] = run["maf_version"]

rubin_sim/maf/web/templates/runselect.html

@@ -41,7 +41,7 @@
   {% for key in runInfo %}
     {% if loop.index == 1 %}
        <td><a href="allMetricResults?runId={{run['maf_run_id']}}">{{runInfo[key]|escape }} </a> </td>
-    {% elif key == 'SQLite File' %}
+    {% elif key == 'RunDb File' %}
        <td><a href="{{runInfo[key][0]}}" download>{{runInfo[key][1]|escape}}</a> </td>
     {% elif key == 'ResultsDb' %}
        <td><a href="{{runInfo[key]}}" download>ResultsDb</a> </td>