DM-45860: Expand support for SIAv2 and VOTable

.github/workflows/do_not_merge.yaml

@@ -0,0 +1,41 @@
+name: "Check commits can be merged"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  do-not-merge-checker:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check that there are no commits that should not be merged
+        uses: gsactions/commit-message-checker@v2
+        with:
+          excludeDescription: "true"  # optional: this excludes the description body of a pull request
+          excludeTitle: "true"  # optional: this excludes the title of a pull request
+          checkAllCommitMessages: "true"  # optional: this checks all commits associated with a pull request
+          accessToken: ${{ secrets.GITHUB_TOKEN }}  # github access token is only required if checkAllCommitMessages is true
+          # Check for message indicating that there is a commit that should
+          # not be merged.
+          pattern: ^(?!DO NOT MERGE)
+          flags: "i"
+          error: |
+            "This step failed because there is a commit containing the text
+            'DO NOT MERGE'. Remove this commit from the branch before merging
+            or change the commit summary."
+
+      - uses: actions/checkout@v4
+
+      - name: Check requirements.txt for branches
+        shell: bash
+        run: |
+          FILE="requirements.txt requirements/main.in requirements/test.in"
+          MATCH=tickets/DM-
+          if grep -q $MATCH $FILE
+          then
+            echo "Ticket branches found in $FILE:"
+            grep -n $MATCH $FILE
+            exit 1
+          fi

.gitignore

@@ -1,3 +1,4 @@
+_build.*
 *.o
 *.so
 *.os
@@ -20,3 +21,4 @@ config.log
 .mypy_cache/
 python/lsst/dax/obscore/version.py
 tests/.tests/
+tests/tmp*

configs/ci_hsc_gen3.yaml

@@ -0,0 +1,58 @@
+facility_name: Subaru
+obs_collection: LSST.CI
+collections: ["HSC/runs/ci_hsc"]
+dataset_types:
+  raw:
+    dataproduct_type: image
+    dataproduct_subtype: lsst.raw
+    calib_level: 1
+    obs_id_fmt: "{records[exposure].obs_id}"
+    o_ucd: phot.count
+    access_format: image/fits
+  calexp:
+    dataproduct_type: image
+    dataproduct_subtype: lsst.calexp
+    calib_level: 2
+    obs_id_fmt: "{records[visit].name}"
+    o_ucd: phot.count
+    access_format: image/fits
+  deepCoadd:
+    dataproduct_type: image
+    dataproduct_subtype: lsst.coadd
+    calib_level: 3
+    obs_id_fmt: "{skymap}-{tract}-{patch}"
+    o_ucd: phot.count
+    access_format: image/fits
+extra_columns:
+  lsst_visit:
+    template: "{visit}"
+    type: "int"
+  lsst_detector:
+    template: "{detector}"
+    type: "int"
+  lsst_tract:
+    template: "{tract}"
+    type: "int"
+  lsst_patch:
+    template: "{patch}"
+    type: "int"
+  lsst_band:
+    template: "{band}"
+    type: "string"
+  lsst_filter:
+    template: "{physical_filter}"
+    type: "string"
+spectral_ranges:
+  "HSC-G": [406.0e-9, 545.0e-9]
+  "HSC-R": [543.0e-9, 693.0e-9]
+  "HSC-R2": [542.0e-9, 693.0e-9]
+  "HSC-I": [690.0e-9, 842.0e-9]
+  "HSC-I2": [692.0e-9, 850.0e-9]
+  "HSC-Z": [852.0e-9, 928.0e-9]
+  "HSC-Y": [937.0e-9, 1015.0e-9]
+  "N921": [914.7e-9, 928.1e-9]
+  "g": [406.0e-9, 545.0e-9]
+  "r": [542.0e-9, 693.0e-9]
+  "i": [692.0e-9, 850.0e-9]
+  "z": [852.0e-9, 928.0e-9]
+  "y": [937.0e-9, 1015.0e-9]

configs/dp02.yaml

@@ -58,10 +58,10 @@ extra_columns:
     type: "int"
   lsst_band:
     template: "{band}"
-    type: "str"
+    type: "string"
   lsst_filter:
     template: "{physical_filter}"
-    type: "str"
+    type: "string"
 spectral_ranges:
   "u": [330.0e-9, 400.0e-9]
   "g": [402.0e-9, 552.0e-9]

pyproject.toml

@@ -29,6 +29,8 @@ dependencies = [
     "lsst-utils",
     "lsst-daf-butler",
     "lsst-sphgeom",
+    "lsst-resources",
+    "lsst-felis",
 ]
 dynamic = ["version"]
 
@@ -51,7 +53,7 @@ zip-safe = true
 license-files = ["COPYRIGHT", "LICENSE"]
 
 [tool.setuptools.package-data]
-"lsst.dax.obscore" = ["py.typed",]
+"lsst.dax.obscore" = ["py.typed", "configs/*.yaml"]
 
 [tool.setuptools.dynamic]
 version = { attr = "lsst_versions.get_lsst_version" }

python/lsst/dax/obscore/cli/cmd/commands.py

@@ -59,8 +59,8 @@
 )
 @click.option(
     "--format",
-    help="Output format, one of 'parquet' or 'csv'; default: parquet.",
-    type=click.Choice(["csv", "parquet"]),
+    help="Output format, one of 'parquet', 'votable', or 'csv'; default: parquet.",
+    type=click.Choice(["csv", "parquet", "votable"]),
     default="parquet",
 )
 @dataset_type_option(
@@ -142,3 +142,78 @@
     used after adding obscore support to existing repository.
     """
     script.obscore_update_table(*args, **kwargs)
+
+
+@obscore.command(
+    short_help="Run SIAv2 query and return results.",
+    cls=ButlerCommand,
+)
+@repo_argument(required=True)
+@destination_argument(
+    required=True,
+    help="DESTINATION is the location of the output file.",
+    type=MWPath(file_okay=True, dir_okay=False, writable=True),
+)
+@click.option(
+    "--config",
+    "-c",
+    help="Location of the configuration file in YAML format, path or URL.",
+    required=True,
+)
+@dataset_type_option(
+    help=(
+        "Comma-separated list of Butler dataset types. "
+        "If specified it must be a subset of dataset types defined in configuration file."
+    )
+)
+@collections_option(
+    help="Butler collections (not SIAv2 collections) to include in the query. Default is to use the "
+    "collections specified in the SIAv2 configuration file."
+)
+@click.option(
+    "--instrument",
+    help="Name of instrument to use in query. If no instrument is specified all instruments are included.",
+    type=str,
+    multiple=True,
+)
+@click.option(
+    "--pos",
+    help="IVOA POS region to use to restrict results. CIRCLE, RANGE and POLYGON are supported.",
+    type=str,
+    multiple=True,
+)
+@click.option(
+    "--time",
+    help="A moment in time or a time span as a range to use to constrain the query. Uses MJD UTC.",
+    type=str,
+    multiple=True,
+)
+@click.option("--band", help="Wavelength range to constrain query. Units of meters.", type=str, multiple=True)
+@click.option(
+    "--exptime",
+    help="Exposure time ranges in seconds.",
+    type=str,
+    multiple=True,
+)
+@click.option(
+    "--calib",
+    help="Calibration level of the data. Allowed values are 0, 1, 2, and 3",
+    multiple=True,
+    type=int,
+)
+@click.option(
+    "--maxrec",
+    help="Maximum number of records to return. 0 means no records.",
+    type=int,
+)
+@options_file_option()
+def siav2(*args: Any, **kwargs: Any) -> None:
+    """Export Butler datasets as ObsCore Data Model in parquet format.
+
+    For details on the SIAv2 parameters see https://www.ivoa.net/documents/SIA/
+
+    Multiple values can be specified for a single parameter and the results are
+    ORed together. All range parameters allow numbers and include -Inf and
+    +Inf as options.
+    """
+    script.obscore_siav2(*args, **kwargs)

python/lsst/dax/obscore/config.py

@@ -21,18 +21,71 @@
 
 from __future__ import annotations
 
-__all__ = ["ExporterConfig"]
+__all__ = ["ExporterConfig", "WhereBind"]
+
+from collections.abc import Iterable
+from typing import Any, Literal
 
 from lsst.daf.butler.registry.obscore import ObsCoreConfig
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class WhereBind(BaseModel):
+    """A where expression with associated bind parameters."""
+
+    model_config = ConfigDict(frozen=True)
+
+    where: str = ""
+    """User expression to restrict the output."""
+    bind: dict[str, Any] = Field(default_factory=dict)
+    """Bind values specified in the ``where`` expression."""
+    extra_dims: frozenset[str] = Field(default_factory=frozenset)
+    """Extra dimensions required to be included in query."""
+
+    @classmethod
+    def combine(cls, wheres: list[WhereBind], mode: Literal["AND"] | Literal["OR"] = "AND") -> WhereBind:
+        """Combine multiple clauses into a single where expression.
+
+        Parameters
+        ----------
+        wheres : `list` [ `WhereBind`]
+            The user expressions to combine.
+        mode : `str`
+            Combination mode. Can be ``AND`` or ``OR``.
+
+        Returns
+        -------
+        combo : `WhereBind`
+            A new `WhereBind` representing all the information of the input
+            clauses.
+        """
+        if len(wheres) == 1:
+            return wheres[0]
+        where = f" {mode} ".join(f"({w.where})" for w in wheres)
+        bind: dict[str, Any] = {}
+        extras: set[str] = set()
+        for w in wheres:
+            # Warn if we are overwriting bind keys.
+            duplicates = bind.keys() & w.bind.keys()
+            if duplicates:
+                raise ValueError(
+                    f"Combining multiple WHERE clauses with reused bind parameters of {duplicates}"
+                )
+            bind.update(w.bind)
+            extras.update(w.extra_dims)
+        return cls(where=where, bind=bind, extra_dims=extras)
 
 
 class ExporterConfig(ObsCoreConfig):
     """Complete configuration for ObscoreExporter."""
 
-    where: str = ""
-    """User expression to restrict the output. This value can be overridden
-    with command line options.
-    """
+    where: WhereBind = Field(default_factory=WhereBind)
+    """Default user expression to restrict the output. Not used if
+    per dataset type user expression is provided."""
+
+    dataset_type_constraints: dict[str, list[WhereBind]] = Field(default_factory=dict)
+    """Specific user expressions for a given dataset type. If a dataset type
+    is not specified here the default ``where`` will be used."""
 
     batch_size: int = 10_000
     """Number of records in a pyarrow RecordBatch"""
@@ -42,3 +95,21 @@
 
     csv_null_string: str = r"\N"
     """Value to use for NULLs in CSV output."""
+
+    def select_dataset_types(self, dataset_types: Iterable[str]) -> None:
+        """Update the configuration to include only these dataset types.
+
+        Parameters
+        ----------
+        dataset_types : `~collections.abc.Iterable` [ `str` ]
+            Names of dataset types to select.
+        """
+        dataset_type_set = set(dataset_types)
+        # Check that configuration has all requested dataset types.
+        if not dataset_type_set.issubset(self.dataset_types):
+            extras = dataset_type_set - set(self.dataset_types)
+            raise ValueError(f"Dataset types {extras} are not defined in configuration file.")
+        # Remove dataset types that are not needed.
+        self.dataset_types = {
+            key: value for key, value in self.dataset_types.items() if key in dataset_type_set
+        }