Add the ability to pull in ERDDAP datasets

src/app.py

@@ -26,9 +26,8 @@
     data_source = st.selectbox("Select data source", DATA_SOURCES)
 
 
-config = {}
 qartod = {}
-
+column = "navd88_meters"
 if data_source == HOHONU_DATA_SOURCE:
     from hohonu_api import load_hohonu_streamlit_data_and_config
 
@@ -39,13 +38,18 @@
 
     data, config = load_things_streamlit_data_and_config()
 
+
+elif data_source == ERDDAP:
+    from erddap import load_erddap_data_and_config
+
+    data, config, column = load_erddap_data_and_config()
 else:
     st.error("No idea how you selected another data source, exploding now")
     st.stop()
 
 with st.expander("Loaded data"):
     st.dataframe(data)
-    st.line_chart(data, x="time", y="navd88_meters", y_label="navd88_meters (m)")
+    st.line_chart(data, x="time", y=column, y_label=column)
 
 with st.sidebar:
     with st.expander("Datum selector", expanded=True):
@@ -143,7 +147,7 @@
                 data,
                 qc_helpers.Config(
                     {
-                        "navd88_meters": {
+                        column: {
                             "qartod": gross_range_test_config,
                         }
                     }
@@ -154,6 +158,7 @@
                 gross_df,
                 "gross_range_test",
                 title="Gross range test",
+                var_name=column,
             )
             st.bokeh_chart(plot, use_container_width=True)
 
@@ -192,7 +197,7 @@
                 data,
                 qc_helpers.Config(
                     {
-                        "navd88_meters": {
+                        column: {
                             "qartod": rate_of_change_test_config,
                         }
                     }
@@ -244,7 +249,7 @@
                 data,
                 qc_helpers.Config(
                     {
-                        "navd88_meters": {
+                        column: {
                             "qartod": spike_test_config,
                         }
                     }
@@ -300,7 +305,7 @@
                 data,
                 qc_helpers.Config(
                     {
-                        "navd88_meters": {
+                        column: {
                             "qartod": flat_line_test_config,
                         }
                     }
@@ -321,7 +326,7 @@
         data,
         qc_helpers.Config(
             {
-                "navd88_meters": {
+                column: {
                     "qartod": qartod,
                 }
             }
@@ -371,7 +376,7 @@
 
     config.update({
         "datums": {"manual_datums": datums},
-        "qc": {"qartod": {"contexts": [{"streams": {"navd88_meters": {"qartod": qartod}}}]}},
+        "qc": {"qartod": {"contexts": [{"streams": {column: {"qartod": qartod}}}]}},
     })
 
     if title := st.text_input("Station title", help="For display in ERDDAP, Mariners, and other locations, such as 'Department of Marine Resources, Boothbay Harbor, ME Hohonu tide gauge'"):

src/erddap.py

@@ -0,0 +1,71 @@
+from datetime import datetime, date, timedelta, time
+
+import erddapy
+import pandas as pd
+
+def load_erddap_data_and_config():
+    import streamlit as st
+
+    config = {}
+
+    with st.sidebar:
+        server_url = st.text_input("ERDDAP URL", "https://data.neracoos.org/erddap")
+        dataset_id = st.text_input("Dataset ID")
+
+    if not dataset_id:
+        st.warning("Please enter a dataset ID")
+        st.stop()
+
+    with st.sidebar:
+        with st.expander("Data selector", expanded=True):
+            now = datetime.now()
+            week_ago = now - timedelta(days=7)
+
+            date_range = st.date_input(
+                "Date range",
+                (week_ago, now),
+                max_value=now,
+                help="""
+                    Date range to load testing data for.
+                    """
+            )
+
+            load_data_button = st.toggle("Load data")
+
+    if not load_data_button:
+        st.warning("Please select a date range and toggle 'Load data' to start generating QARTOD config")
+        st.stop()
+
+    @st.cache_data
+    def fetch_data(server_url, dataset_id, date_range):
+
+        e = erddapy.ERDDAP(server=server_url, protocol="tabledap", response="csv")
+        e.dataset_id = dataset_id
+        e.constraints = {"time>=": date_range[0].isoformat(), "time<=": date_range[1].isoformat()}
+
+        df = e.to_pandas(parse_dates=True)
+
+        return df
+
+    df = fetch_data(server_url, dataset_id, date_range)
+
+    with st.sidebar:
+        with st.expander("Rename columns", expanded=False):
+            st.markdown("""
+                Rename columns
+                - Usually units need to be removed from column names
+                - Existing QARTOD test columns should be renamed
+                        """)
+            rename_columns = st.data_editor(
+                pd.DataFrame({"source_column": ["time (UTC)", "navd88_meters (m)"],  "target_column": ["time", "navd88_meters"]}),
+                num_rows="dynamic"
+            )
+            rename_map = {row["source_column"]: row["target_column"] for _, row in rename_columns.iterrows()}
+            df = df.rename(columns=rename_map)
+
+        df["time"] = pd.to_datetime(df["time"])
+
+        column = st.selectbox("Select a column to compute QARTOD for", df.columns)
+
+
+    return df, config, column

src/qc_helpers.py

@@ -28,9 +28,21 @@ def plot_results(
 
     time = data["time"]
     obs = data[var_name]
-    qc_test = data[f"{var_name}_qartod_{test_name}"]
-
-    qc_pass = np.ma.masked_where(qc_test != 1, obs)
+    qc_test_column = f"{var_name}_qartod_{test_name}"
+    try:
+        qc_test = data[qc_test_column]
+    except KeyError as e:
+        raise KeyError(
+            f"{qc_test_column} is not in the dataframe. {data.columns}"
+        ) from e
+
+    try:
+        qc_pass = np.ma.masked_where(qc_test != 1, obs)
+    except IndexError as e:
+        raise IndexError(
+            f"{qc_test=}, {obs=}"
+        ) from e
+
     qc_suspect = np.ma.masked_where(qc_test != 3, obs)
     qc_fail = np.ma.masked_where(qc_test != 4, obs)
     qc_notrun = np.ma.masked_where(qc_test != 2, obs)