Various endpoints (#30)

* Add new /schema endpoint

* Fix URL

* Add skymap route (formerly /bayestar)

* Fix wrong import

* Fix wrong class

* Fix tests

* Add new /statistics route

* TODO

* New /ssocand route

* Reduce the number of entries in Swagger

* For VOTable, the MIME type of the response should be text/xml.

* New /anomaly route

* Add the /ssoft route

* Add args for the ssoft

* Update profiling

* Fix schema

* PEP8

* Default parquet for Swagger

* Fix logic

* Fi schema test

* Fix spelling

* Fix return types

* Fix typo

* Missing import

* Fix return type for schema

* Update README

* Factorize code

* New /metadata route

* Add route

* PEP8

* Fix typo

* Fix synatx

README.md

@@ -138,7 +138,5 @@ You find a [template](apps/routes/template) route to start a new route. Just cop
 ## Todo
 
 - [ ] configuration: Find a way to automatically sync schema with tables.
-- [ ] Add nginx management
-- [ ] Add bash scripts under `bin/` to manage both nginx and gunicorn
 - [ ] Make tests more verbose, even is successful.
 

app.py

@@ -27,6 +27,13 @@
 from apps.routes.v1.sso.api import ns as ns_sso
 from apps.routes.v1.resolver.api import ns as ns_resolver
 from apps.routes.v1.tracklet.api import ns as ns_tracklet
+from apps.routes.v1.schema.api import ns as ns_schema
+from apps.routes.v1.skymap.api import ns as ns_skymap
+from apps.routes.v1.statistics.api import ns as ns_statistics
+from apps.routes.v1.ssocand.api import ns as ns_ssocand
+from apps.routes.v1.anomaly.api import ns as ns_anomaly
+from apps.routes.v1.ssoft.api import ns as ns_ssoft
+from apps.routes.v1.metadata.api import ns as ns_metadata
 
 config = extract_configuration("config.yml")
 
@@ -63,8 +70,15 @@ def after_request(response):
 api.add_namespace(ns_classes)
 api.add_namespace(ns_conesearch)
 api.add_namespace(ns_sso)
+api.add_namespace(ns_ssocand)
 api.add_namespace(ns_resolver)
 api.add_namespace(ns_tracklet)
+api.add_namespace(ns_schema)
+api.add_namespace(ns_skymap)
+api.add_namespace(ns_statistics)
+api.add_namespace(ns_anomaly)
+api.add_namespace(ns_ssoft)
+api.add_namespace(ns_metadata)
 
 # Register blueprint
 app.register_blueprint(blueprint)

apps/routes/v1/anomaly/api.py

@@ -0,0 +1,89 @@
+# Copyright 2024 AstroLab Software
+# Author: Julien Peloton
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from flask import Response, request
+from flask_restx import Namespace, Resource, fields
+
+from apps.utils.utils import check_args
+from apps.utils.utils import send_tabular_data
+
+from apps.routes.v1.anomaly.utils import get_anomalous_alerts
+
+ns = Namespace("api/v1/anomaly", "Get alerts tagged as anomaly")
+
+ARGS = ns.model(
+    "anomaly",
+    {
+        "n": fields.Integer(
+            description="Last N alerts to transfer between stop and start date (going from most recent to older alerts). Default is 10.",
+            example=10,
+            required=False,
+        ),
+        "start_date": fields.String(
+            description="[Optional] Start date in UTC YYYY-MM-DD. Default is 2019-11-01.",
+            example="2019-11-01",
+            required=False,
+        ),
+        "stop_date": fields.String(
+            description="[Optional] Stop date in UTC YYYY-MM-DD. Default is now.",
+            required=False,
+        ),
+        "columns": fields.String(
+            description="Comma-separated data columns to transfer, e.g. 'i:magpsf,i:jd'. If not specified, transfer all columns.",
+            example="i:jd,i:magpsf,i:fid",
+            required=False,
+        ),
+        "output-format": fields.String(
+            description="Output format among json[default], csv, parquet, votable.",
+            example="json",
+            required=False,
+        ),
+    },
+)
+
+
+@ns.route("")
+@ns.doc(params={k: ARGS[k].description for k in ARGS})
+class Anomaly(Resource):
+    def get(self):
+        """Retrieve alerts tagged as anomaly from the Fink/ZTF database"""
+        payload = request.args
+        if len(payload) > 0:
+            # POST from query URL
+            return self.post()
+        else:
+            return Response(ns.description, 200)
+
+    @ns.expect(ARGS, location="json", as_dict=True)
+    def post(self):
+        """Retrieve alerts tagged as anomaly from the Fink/ZTF database"""
+        # get payload from the query URL
+        payload = request.args
+
+        if payload is None or len(payload) == 0:
+            # if no payload, try the JSON blob
+            payload = request.json
+
+        rep = check_args(ARGS, payload)
+        if rep["status"] != "ok":
+            return Response(str(rep), 400)
+
+        out = get_anomalous_alerts(payload)
+
+        # Error propagation
+        if isinstance(out, Response):
+            return out
+
+        output_format = payload.get("output-format", "json")
+        return send_tabular_data(out, output_format)

apps/routes/v1/anomaly/profiling.py

@@ -0,0 +1,23 @@
+# Copyright 2024 AstroLab Software
+# Author: Julien Peloton
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Call get_anomalous_alerts"""
+
+from apps.routes.v1.anomaly.utils import get_anomalous_alerts
+
+payload = {
+    "n": 10000,
+}
+
+get_anomalous_alerts(payload)

apps/routes/v1/anomaly/test.py

@@ -0,0 +1,168 @@
+# Copyright 2023 AstroLab Software
+# Author: Julien Peloton
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import requests
+import pandas as pd
+import numpy as np
+
+from astropy.io import votable
+
+import io
+import sys
+import json
+
+APIURL = sys.argv[1]
+
+
+def anomalysearch(
+    n=10, start_date=None, stop_date=None, output_format="json", cols=None
+):
+    """Perform a search for anomaly"""
+    payload = {"n": n, "output-format": output_format}
+
+    if start_date is not None:
+        payload.update({"start_date": start_date, "stop_date": stop_date})
+
+    if cols is not None:
+        payload.update({"columns": cols})
+
+    r = requests.post("{}/api/v1/anomaly".format(APIURL), json=payload)
+
+    assert r.status_code == 200, r.content
+
+    if output_format == "json":
+        # Format output in a DataFrame
+        pdf = pd.read_json(io.BytesIO(r.content))
+    elif output_format == "csv":
+        pdf = pd.read_csv(io.BytesIO(r.content))
+    elif output_format == "parquet":
+        pdf = pd.read_parquet(io.BytesIO(r.content))
+    elif output_format == "votable":
+        vt = votable.parse(io.BytesIO(r.content))
+        pdf = vt.get_first_table().to_table().to_pandas()
+
+    return pdf
+
+
+def test_simple_anomaly() -> None:
+    """
+    Examples
+    --------
+    >>> test_simple_anomaly()
+    """
+    pdf = anomalysearch()
+
+    assert not pdf.empty
+
+    assert len(pdf) == 10, len(pdf)
+
+    assert np.all(pdf["d:anomaly_score"].to_numpy() < 0)
+
+
+def test_anomaly_and_date() -> None:
+    """
+    Examples
+    --------
+    >>> test_anomaly_and_date()
+    """
+    pdf = anomalysearch(start_date="2023-01-25", stop_date="2023-01-25")
+
+    assert not pdf.empty
+
+    assert len(pdf) == 10, len(pdf)
+
+    assert "ZTF23aaaatwl" in pdf["i:objectId"].to_numpy()
+
+
+def test_anomaly_and_cols_with_sort() -> None:
+    """
+    Examples
+    --------
+    >>> test_anomaly_and_cols_with_sort()
+    """
+    pdf = anomalysearch(cols="i:jd,i:objectId")
+
+    assert not pdf.empty
+
+    assert len(pdf.columns) == 2, len(pdf.columns)
+
+    assert "i:jd" in pdf.columns
+    assert "i:objectId" in pdf.columns
+    assert "v:classifation" not in pdf.columns
+
+
+def test_query_url() -> None:
+    """
+    Examples
+    --------
+    >>> test_query_url()
+    """
+    pdf1 = anomalysearch()
+
+    url = "{}/api/v1/anomaly?n=10&output-format=json".format(APIURL)
+    r = requests.get(url)
+    pdf2 = pd.read_json(io.BytesIO(r.content))
+
+    # subset of cols to avoid type issues
+    cols = ["d:anomaly_score"]
+
+    isclose = np.isclose(pdf1[cols], pdf2[cols])
+    assert np.all(isclose)
+
+
+def test_various_outputs() -> None:
+    """
+    Examples
+    --------
+    >>> test_various_outputs()
+    """
+    pdf1 = anomalysearch(output_format="json")
+
+    for fmt in ["csv", "parquet", "votable"]:
+        pdf2 = anomalysearch(output_format=fmt)
+
+        # subset of cols to avoid type issues
+        cols1 = ["d:anomaly_score"]
+
+        # https://docs.astropy.org/en/stable/io/votable/api_exceptions.html#w02-x-attribute-y-is-invalid-must-be-a-standard-xml-id
+        cols2 = cols1 if fmt != "votable" else ["d_anomaly_score"]
+
+        isclose = np.isclose(pdf1[cols1], pdf2[cols2])
+        assert np.all(isclose), fmt
+
+
+def test_feature_array() -> None:
+    """
+    Examples
+    --------
+    >>> test_feature_array()
+    """
+    pdf = anomalysearch()
+
+    a_feature = pdf["d:lc_features_g"].to_numpy()[0]
+    assert isinstance(a_feature, str), a_feature
+
+    for col in ["d:lc_features_g", "d:lc_features_r"]:
+        pdf[col] = pdf[col].apply(lambda x: json.loads(x))
+
+    a_feature = pdf["d:lc_features_g"].to_numpy()[0]
+    assert isinstance(a_feature, list), a_feature
+
+
+if __name__ == "__main__":
+    """ Execute the test suite """
+    import sys
+    import doctest
+
+    sys.exit(doctest.testmod()[0])