Enable logging and tracing (diagnostics) for opscenter (#170)

* Adds instrumentation to track basic usage
* Adds some structured logging for obvious errors (tasks)
* Enables log_level=INFO, trace=ON_EVENT for app manifest
* Adds a tab to Settings page to explain how to enable diagnostics

Closes #57

app/manifest.yml

@@ -5,6 +5,10 @@ artifacts:
   extension_code: true
   default_streamlit: admin.opscenter
 
+configuration:
+  trace_level: ON_EVENT
+  log_level: INFO
+
 privileges:
   - IMPORTED PRIVILEGES ON SNOWFLAKE DB:
       description: "For accessing Snowflake consumption for reporting."

app/ui/labels.py

@@ -13,6 +13,11 @@ def __init__(self):
         self.snowflake = Connection.get()
 
     def list_labels(self):
+        _ = self.snowflake.call(
+            "INTERNAL.REPORT_ACTION",
+            "labels",
+            "list",
+        )
         st.title("Query Labels")
         st.write(
             """
@@ -136,6 +141,11 @@ def list_labels(self):
 
     def on_create_click(self, name, group, rank, condition):
         with st.spinner("Creating new label..."):
+            _ = self.snowflake.call(
+                "INTERNAL.REPORT_ACTION",
+                "labels",
+                "create",
+            )
             outcome = self.snowflake.call(
                 "ADMIN.CREATE_LABEL", name, group, rank, condition
             )
@@ -150,6 +160,12 @@ def on_create_click(self, name, group, rank, condition):
     def on_update_click(self, oldname, name, group, rank, condition):
         outcome = None
         with st.spinner("Updating label..."):
+            _ = self.snowflake.call(
+                "INTERNAL.REPORT_ACTION",
+                "labels",
+                "update",
+            )
+
             outcome = self.snowflake.call(
                 "ADMIN.UPDATE_LABEL", oldname, name, group, rank, condition
             )
@@ -162,6 +178,11 @@ def on_update_click(self, oldname, name, group, rank, condition):
 
     def on_delete_click(self, name):
         with st.spinner("Deleting label..."):
+            _ = self.snowflake.call(
+                "INTERNAL.REPORT_ACTION",
+                "labels",
+                "delete",
+            )
             self.snowflake.call("ADMIN.DELETE_LABEL", name)
             self.session.set_toast("Label deleted.")
             self.session.do_list()

app/ui/pages/10_Settings.py

@@ -51,8 +51,8 @@ def task_listing(
     )
 
 
-tasks, config_tab, setup_tab, reset = st.tabs(
-    ["Tasks", "Config", "Initial Setup", "Reset"]
+tasks, config_tab, setup_tab, diagnostics_tab, reset = st.tabs(
+    ["Tasks", "Config", "Initial Setup", "Diagnostics", "Reset"]
 )
 
 
@@ -159,6 +159,71 @@ def save_tasks(container, wem, qhm, pm):
             "Unable to load task information. Make sure to run post-setup scripts."
         )
 
+with diagnostics_tab:
+    st.title("Diagnostics")
+
+    st.markdown(
+        """
+        Diagnostics for OpsCenter relies on an [Snowflake Event Table](https://docs.snowflake.com/en/developer-guide/logging-tracing/event-table-setting-up)
+        to store OpsCenter logging in your Snowflake Account as well as share OpsCenter errors with Sundeck to fix.
+
+        Snowflake will automatically share logging and errors from OpsCenter with Sundeck after executing these steps.
+        """
+    )
+
+    db = connection.execute("select current_database() as db").values[0][0]
+
+    def expander(num: int, title: str) -> st.expander:
+        return st.expander(f"Step {num}: {title}", expanded=True)
+
+    # We can't inspect the account parameters to see if an event table is set because we're operating as the native
+    # app and these commands cannot be run except as a caller. A human has to run these commands.
+    with expander(1, "Create and Configure an Event Table"):
+        st.markdown(
+            """
+            ### Event Table
+
+            If you haven't already configured an event table for your account, follow these steps:
+
+            These commands will create an event table and set it as the default event
+            table for your account. Be sure to include use a database and schema that exists in your account.
+            """
+        )
+        st.code(
+            """
+            -- Double check that there is no event table already set for your account before proceeding!
+            SHOW PARAMETERS LIKE 'EVENT_TABLE' IN ACCOUNT;
+
+            -- Create a database
+            CREATE DATABASE my_database;
+
+            -- Create the event table in that database
+            CREATE EVENT TABLE my_database.public.my_events;
+
+            -- Set this event table as the default for your account
+            ALTER ACCOUNT SET EVENT_TABLE = my_database.public.my_events;
+            """
+        )
+        st.markdown(
+            """
+            You can also follow the [Snowflake instructions](https://docs.snowflake.com/en/developer-guide/logging-tracing/event-table-setting-up) to
+            set up an event table if you prefer.
+            """
+        )
+
+    with expander(2, "Enable Diagnostic Sharing with Sundeck for OpsCenter"):
+        st.markdown(
+            """
+            Sharing diagnostics with Sundeck helps us know when users are experiencing any errors in OpsCenter
+            so we can fix them as soon as possible. To enable this, please run the following:
+            """
+        )
+        st.code(
+            f"""
+            ALTER APPLICATION {db} SET SHARE_EVENTS_WITH_PROVIDER = true;
+            """
+        )
+
 
 with reset:
     st.title("Reset/Reload")

app/ui/probes.py

@@ -29,6 +29,11 @@ def __init__(self):
         self.snowflake = Connection.get()
 
     def list_probes(self):
+        _ = self.snowflake.call(
+            "INTERNAL.REPORT_ACTION",
+            "probes",
+            "list",
+        )
         st.title("Query Probes")
         st.markdown(
             """
@@ -132,6 +137,11 @@ def on_create_click(
         cancel,
     ):
         with st.spinner("Creating new probe..."):
+            _ = self.snowflake.call(
+                "INTERNAL.REPORT_ACTION",
+                "probes",
+                "create",
+            )
             outcome = self.snowflake.call(
                 "ADMIN.CREATE_PROBE",
                 name,
@@ -163,6 +173,11 @@ def on_update_click(
     ):
         outcome = None
         with st.spinner("Updating probe..."):
+            _ = self.snowflake.call(
+                "INTERNAL.REPORT_ACTION",
+                "probes",
+                "update",
+            )
             outcome = self.snowflake.call(
                 "ADMIN.UPDATE_PROBE",
                 oldname,
@@ -184,6 +199,11 @@ def on_update_click(
 
     def on_delete_click(self, name):
         with st.spinner("Deleting probe..."):
+            _ = self.snowflake.call(
+                "INTERNAL.REPORT_ACTION",
+                "probes",
+                "delete",
+            )
             self.snowflake.call("ADMIN.DELETE_PROBE", name)
             self.session.set_toast("Probe deleted.")
             self.session.do_list()

app/ui/reports_dbt.py

@@ -12,6 +12,8 @@ def report(
     bf: filters.BaseFilter,
     cost_per_credit,
 ):
+    _ = connection.execute("CALL INTERNAL.REPORT_PAGE_VIEW('Query Report dbt Summary')")
+
     labels = connection.execute_with_cache(
         "select name from internal.labels where group_name is null"
     )

app/ui/reports_heatmap.py

@@ -11,6 +11,7 @@ def heatmap(
     bf: filters.BaseFilter,
     cost_per_credit,
 ):
+    _ = connection.execute("CALL INTERNAL.REPORT_PAGE_VIEW('Warehouse Heatmap')")
 
     sql = f"""
     with util as (

app/ui/reports_query_activity.py

@@ -75,6 +75,10 @@ def report(
     else:
         grp = f""" "{grouping}" """
 
+    _ = connection.execute(
+        f"CALL INTERNAL.REPORT_PAGE_VIEW('Query Activity by {grouping}')"
+    )
+
     def overview():
         sql = f"""
         select

app/ui/reports_top_spenders.py

@@ -10,6 +10,10 @@ def report(
     bf: filters.BaseFilter,
     cost_per_credit,
 ):
+    _ = connection.execute(
+        "CALL INTERNAL.REPORT_PAGE_VIEW('Query Report Top Spenders')"
+    )
+
     labels = connection.execute_with_cache(
         "select name from internal.labels where group_name is null"
     )

app/ui/reports_warehouse.py

@@ -178,6 +178,8 @@ def warehouse_users(container):
 
         st.plotly_chart(fig, use_container_width=True)
 
+    _ = connection.execute("CALL INTERNAL.REPORT_PAGE_VIEW('Warehouse Activity')")
+
     warehouse_stats(st.empty())
     cols = st.columns(2)
     with cols[0]:

bootstrap/018_warehouse_updates.sql

@@ -6,11 +6,11 @@ CREATE OR REPLACE PROCEDURE internal.refresh_warehouse_events(migrate boolean) R
     AS
 BEGIN
     let dt timestamp := current_timestamp();
-    SYSTEM$LOG_INFO('Starting refresh queries.');
+    SYSTEM$LOG_INFO('Starting refresh warehouse events.');
     let migrate1 string := null;
     let migrate2 string := null;
     if (migrate) then
-        SYSTEM$LOG_TRACE('Migrating data.');
+        SYSTEM$LOG_TRACE('Migrating warehouse events data.');
         call internal.migrate_if_necessary('INTERNAL_REPORTING', 'CLUSTER_AND_WAREHOUSE_SESSIONS_COMPLETE_AND_DAILY', 'INTERNAL_REPORTING_MV', 'CLUSTER_AND_WAREHOUSE_SESSIONS_COMPLETE_AND_DAILY');
         migrate1 := (select * from TABLE(RESULT_SCAN(LAST_QUERY_ID())));
         call internal.migrate_if_necessary('INTERNAL_REPORTING', 'CLUSTER_AND_WAREHOUSE_SESSIONS_COMPLETE_AND_DAILY', 'INTERNAL_REPORTING_MV', 'CLUSTER_AND_WAREHOUSE_SESSIONS_COMPLETE_AND_DAILY_INCOMPLETE');
@@ -57,7 +57,7 @@ BEGIN
 
     EXCEPTION
       WHEN OTHER THEN
-        SYSTEM$LOG_ERROR('Exception occurred.', OBJECT_CONSTRUCT('Error type', 'Other error', 'SQLCODE', :sqlcode, 'SQLERRM', :sqlerrm, 'SQLSTATE', :sqlstate));
+        SYSTEM$LOG_ERROR(OBJECT_CONSTRUCT('error', 'Exception occurred while refreshing warehouse events.', 'SQLCODE', :sqlcode, 'SQLERRM', :sqlerrm, 'SQLSTATE', :sqlstate));
         ROLLBACK;
         insert into INTERNAL.TASK_WAREHOUSE_EVENTS SELECT :dt, false, :input, OBJECT_CONSTRUCT('Error type', 'Other error', 'SQLCODE', :sqlcode, 'SQLERRM', :sqlerrm, 'SQLSTATE', :sqlstate)::variant;
         RAISE;

bootstrap/019_query_history_updates.sql

@@ -8,7 +8,7 @@ BEGIN
     let migrate1 string := null;
     let migrate2 string := null;
     if (migrate) then
-        SYSTEM$LOG_TRACE('Migrating data.');
+        SYSTEM$LOG_TRACE('Migrating query history data.');
         call internal.migrate_view();
         call internal.migrate_if_necessary('INTERNAL_REPORTING', 'QUERY_HISTORY_COMPLETE_AND_DAILY', 'INTERNAL_REPORTING_MV', 'QUERY_HISTORY_COMPLETE_AND_DAILY');
         migrate1 := (select * from TABLE(RESULT_SCAN(LAST_QUERY_ID())));
@@ -56,7 +56,7 @@ BEGIN
 
     EXCEPTION
       WHEN OTHER THEN
-        SYSTEM$LOG_ERROR('Exception occurred.', OBJECT_CONSTRUCT('Error type', 'Other error', 'SQLCODE', :sqlcode, 'SQLERRM', :sqlerrm, 'SQLSTATE', :sqlstate));
+        SYSTEM$LOG_ERROR(OBJECT_CONSTRUCT('error', 'Exception occurred while refreshing query history.', 'SQLCODE', :sqlcode, 'SQLERRM', :sqlerrm, 'SQLSTATE', :sqlstate));
         ROLLBACK;
         insert into INTERNAL.TASK_QUERY_HISTORY SELECT :dt, false, :input, OBJECT_CONSTRUCT('Error type', 'Other error', 'SQLCODE', :sqlcode, 'SQLERRM', :sqlerrm, 'SQLSTATE', :sqlstate)::variant;
         RAISE;

bootstrap/040_diagnostics.sql

@@ -0,0 +1,18 @@
+
+CREATE OR REPLACE PROCEDURE INTERNAL.REPORT_PAGE_VIEW(page string)
+RETURNS text
+LANGUAGE SQL
+AS
+BEGIN
+    SYSTEM$LOG_INFO(OBJECT_CONSTRUCT('action', 'page_view', 'page', page));
+    return '';
+END;
+
+CREATE OR REPLACE PROCEDURE INTERNAL.REPORT_ACTION(domain string, verb string)
+RETURNS text
+LANGUAGE SQL
+AS
+BEGIN
+    SYSTEM$LOG_INFO(OBJECT_CONSTRUCT('action', verb, 'domain', domain));
+    return '';
+END;

bootstrap/090_post_setup.sql

@@ -139,6 +139,11 @@ Query Text: {bt}{bt}{bt}{query_text}{bt}{bt}{bt}
         let query_id string := act.QUERY_ID;
         insert into internal.probe_actions select CURRENT_TIMESTAMP(), :name, :query_id, parse_json(:action), :outcome;
     END FOR;
+EXCEPTION
+    WHEN OTHER THEN
+        SYSTEM$LOG_ERROR(OBJECT_CONSTRUCT('error', 'Exception occurred during probe monitoring.', 'outcome', :outcome, 'probe_action' :action, 'SQLCODE', :sqlcode, 'SQLERRM', :sqlerrm, 'SQLSTATE', :sqlstate));
+        insert into internal.probe_actions select CURRENT_TIMESTAMP(), :name, :query_id, parse_json(:action), :outcome;
+        RAISE;
 END;
 
 -- This clarifies that the post setup script has been executed to match the current installed version.