Add experiment information to event monitoring (#4519)

mozilla · Nov 7, 2023 · 0ae76d0 · 0ae76d0
1 parent 3915be1
commit 0ae76d0
Show file tree

Hide file tree

Showing 3 changed files with 126 additions and 8 deletions.
diff --git a/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql b/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql
@@ -40,13 +40,40 @@
     "{{ dataset.get('app_channel', 'release') }}" AS normalized_channel,
     {% endif %}
     client_info.app_display_version AS version,
+    -- Access experiment information.
+    -- Additional iteration is necessary to aggregate total event count across experiments
+    -- which is denoted with "*".
+    -- Some clients are enrolled in multiple experiments, so simply summing up the totals 
+    -- across all the experiments would double count events.
+    CASE 
+      experiment_index 
+    WHEN 
+      ARRAY_LENGTH(ping_info.experiments) 
+    THEN 
+      "*" 
+    ELSE 
+      ping_info.experiments[SAFE_OFFSET(experiment_index)].key 
+    END AS experiment,
+    CASE 
+      experiment_index 
+    WHEN 
+      ARRAY_LENGTH(ping_info.experiments) 
+    THEN 
+      "*" 
+    ELSE 
+      ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch 
+    END AS experiment_branch,
     COUNT(*) AS total_events
   FROM
     `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.events_v1`
   CROSS JOIN
     UNNEST(events) AS event,
-    UNNEST(event.extra) AS event_extra
-  WHERE DATE(submission_timestamp) = @submission_date
+    UNNEST(event.extra) AS event_extra,
+    -- Iterator for accessing experiments.
+    -- Add one more for aggregating events across all experiments
+    UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(ping_info.experiments))) AS experiment_index
+  WHERE 
+    DATE(submission_timestamp) = @submission_date
   GROUP BY
     submission_date,
     window_start,
@@ -57,7 +84,9 @@
     country,
     normalized_app_name,
     normalized_channel,
-    version
+    version,
+    experiment,
+    experiment_branch
 {% elif dataset in ["accounts_frontend", "accounts_backend"] %}
   {% if not outer_loop.first -%}
   UNION ALL
@@ -92,11 +121,39 @@
     normalized_country_code AS country,
     "{{ dataset['canonical_app_name'] }}" AS normalized_app_name,
     normalized_channel,
-    client_info.app_display_version AS VERSION,
+    client_info.app_display_version AS version,
+    -- Access experiment information.
+    -- Additional iteration is necessary to aggregate total event count across experiments
+    -- which is denoted with "*".
+    -- Some clients are enrolled in multiple experiments, so simply summing up the totals 
+    -- across all the experiments would double count events.
+    CASE 
+      experiment_index 
+    WHEN 
+      ARRAY_LENGTH(ping_info.experiments) 
+    THEN 
+      "*" 
+    ELSE 
+      ping_info.experiments[SAFE_OFFSET(experiment_index)].key 
+    END AS experiment,
+    CASE 
+      experiment_index 
+    WHEN 
+      ARRAY_LENGTH(ping_info.experiments) 
+    THEN 
+      "*" 
+    ELSE 
+      ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch 
+    END AS experiment_branch,
     COUNT(*) AS total_events
   FROM
     `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.accounts_events_v1`
-  WHERE DATE(submission_timestamp) = @submission_date
+  CROSS JOIN
+    -- Iterator for accessing experiments.
+    -- Add one more for aggregating events across all experiments
+    UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(ping_info.experiments))) AS experiment_index
+  WHERE 
+    DATE(submission_timestamp) = @submission_date
   GROUP BY
     window_start,
     window_end,
@@ -106,7 +163,9 @@
     country,
     normalized_app_name,
     normalized_channel,
-    version
+    version,
+    experiment,
+    experiment_branch
 {% endif %}
 {% endfor %}
 {% endfor %}
diff --git a/sql_generators/glean_usage/templates/event_monitoring_live.view.sql b/sql_generators/glean_usage/templates/event_monitoring_live.view.sql
@@ -13,6 +13,8 @@ SELECT
   normalized_app_name,
   normalized_channel,
   version,
+  experiment,
+  experiment_branch,
   total_events
 FROM 
   `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_derived.event_monitoring_live_v1`
@@ -32,6 +34,8 @@ SELECT
   normalized_app_name,
   normalized_channel,
   version,
+  experiment,
+  experiment_branch,
   total_events
 FROM 
   `{{ project_id }}.{{ target_table }}`

diff --git a/sql_generators/glean_usage/templates/event_monitoring_live_v1.init.sql b/sql_generators/glean_usage/templates/event_monitoring_live_v1.init.sql
@@ -60,12 +60,38 @@ IF
       '{{ app_name }}' AS normalized_app_name,
       normalized_channel,
       client_info.app_display_version AS version,
+      -- Access experiment information.
+      -- Additional iteration is necessary to aggregate total event count across experiments
+      -- which is denoted with "*".
+      -- Some clients are enrolled in multiple experiments, so simply summing up the totals 
+      -- across all the experiments would double count events.
+      CASE 
+        experiment_index 
+      WHEN 
+        ARRAY_LENGTH(ping_info.experiments) 
+      THEN 
+        "*" 
+      ELSE 
+        ping_info.experiments[SAFE_OFFSET(experiment_index)].key 
+      END AS experiment,
+      CASE 
+        experiment_index 
+      WHEN 
+        ARRAY_LENGTH(ping_info.experiments) 
+      THEN 
+        "*" 
+      ELSE 
+        ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch 
+      END AS experiment_branch,
       COUNT(*) AS total_events
     FROM
       `{{ project_id }}.{{ dataset }}_live.events_v1`
     CROSS JOIN
       UNNEST(events) AS event,
-      UNNEST(event.extra) AS event_extra
+      UNNEST(event.extra) AS event_extra,
+      -- Iterator for accessing experiments.
+      -- Add one more for aggregating events across all experiments
+      UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(ping_info.experiments))) AS experiment_index
     {% elif dataset_id in ["accounts_frontend", "accounts_backend"] %}
       -- FxA uses custom pings to send events without a category and extras.
     SELECT
@@ -98,9 +124,36 @@ IF
       '{{ app_name }}' AS normalized_app_name,
       normalized_channel,
       client_info.app_display_version AS VERSION,
+      -- Access experiment information.
+      -- Additional iteration is necessary to aggregate total event count across experiments
+      -- which is denoted with "*".
+      -- Some clients are enrolled in multiple experiments, so simply summing up the totals 
+      -- across all the experiments would double count events.
+      CASE 
+        experiment_index 
+      WHEN 
+        ARRAY_LENGTH(ping_info.experiments) 
+      THEN 
+        "*" 
+      ELSE 
+        ping_info.experiments[SAFE_OFFSET(experiment_index)].key 
+      END AS experiment,
+      CASE 
+        experiment_index 
+      WHEN 
+        ARRAY_LENGTH(ping_info.experiments) 
+      THEN 
+        "*" 
+      ELSE 
+        ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch 
+      END AS experiment_branch,
       COUNT(*) AS total_events
     FROM
       `{{ project_id }}.{{ dataset }}_live.accounts_events_v1`
+    CROSS JOIN
+      -- Iterator for accessing experiments.
+      -- Add one more for aggregating events across all experiments
+      UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(ping_info.experiments))) AS experiment_index
     {% endif %}
     WHERE
       DATE(submission_timestamp) >= "{{ current_date }}"
@@ -114,4 +167,6 @@ IF
       country,
       normalized_app_name,
       normalized_channel,
-      version
+      version,
+      experiment,
+      experiment_branch