add enrollment_status metric

update changelog

Author: jeddai

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 [Full Changelog](In progress)
 
+## Nimbus SDK ⛅️🔬🔭
+
+### ✨ What's New ✨
+
+- Added the `enrollment_status` metric and defined a host metric callback handler interface ([#5857](https://github.com/mozilla/application-services/pull/5857)).
+
 ## Rust log forwarder
 
 ### 🦊 What's Changed 🦊

components/nimbus/android/src/main/java/org/mozilla/experiments/nimbus/Nimbus.kt

@@ -33,6 +33,8 @@ import org.mozilla.experiments.nimbus.internal.AvailableRandomizationUnits
 import org.mozilla.experiments.nimbus.internal.EnrolledExperiment
 import org.mozilla.experiments.nimbus.internal.EnrollmentChangeEvent
 import org.mozilla.experiments.nimbus.internal.EnrollmentChangeEventType
+import org.mozilla.experiments.nimbus.internal.EnrollmentStatusExtraDef
+import org.mozilla.experiments.nimbus.internal.MetricsHandler
 import org.mozilla.experiments.nimbus.internal.NimbusClient
 import org.mozilla.experiments.nimbus.internal.NimbusClientInterface
 import org.mozilla.experiments.nimbus.internal.NimbusException
@@ -77,6 +79,23 @@ open class Nimbus(
 
     private val logger = delegate.logger
 
+    private val metricsHandler = object : MetricsHandler {
+        override fun recordEnrollmentStatuses(enrollmentStatusExtras: List<EnrollmentStatusExtraDef>) {
+            for (extra in enrollmentStatusExtras) {
+                NimbusEvents.enrollmentStatus.record(
+                    NimbusEvents.EnrollmentStatusExtra(
+                        branch = extra.branch,
+                        slug = extra.slug,
+                        status = extra.status,
+                        reason = extra.reason,
+                        errorString = extra.errorString,
+                        conflictSlug = extra.conflictSlug,
+                    ),
+                )
+            }
+        }
+    }
+
     private val nimbusClient: NimbusClientInterface
 
     override var globalUserParticipation: Boolean
@@ -118,6 +137,7 @@ open class Nimbus(
             // The "dummy" field here is required for obscure reasons when generating code on desktop,
             // so we just automatically set it to a dummy value.
             AvailableRandomizationUnits(clientId = null, userId = null, nimbusId = null, dummy = 0),
+            metricsHandler,
         )
     }
 

components/nimbus/android/src/test/java/org/mozilla/experiments/nimbus/NimbusTests.kt

@@ -644,6 +644,31 @@ class NimbusTests {
 
         assertTrue(observed)
     }
+
+    @Test
+    fun `Nimbus records EnrollmentStatus metrics`() {
+        suspend fun getString(): String {
+            return testExperimentsJsonString(appInfo, packageName)
+        }
+
+        val job = nimbus.applyLocalExperiments(::getString)
+        runBlocking {
+            job.join()
+        }
+
+        assertEquals(1, nimbus.getAvailableExperiments().size)
+        assertNotNull("Event must have a value", NimbusEvents.enrollmentStatus.testGetValue())
+        val enrollmentStatusEvents = NimbusEvents.enrollmentStatus.testGetValue()!!
+        assertEquals("Event count must match", enrollmentStatusEvents.count(), 1)
+
+        val enrolledExtra = enrollmentStatusEvents[0].extra!!
+        assertEquals("branch must match", "test-branch", enrolledExtra["branch"])
+        assertEquals("slug must match", "test-experiment", enrolledExtra["slug"])
+        assertEquals("status must match", "Enrolled", enrolledExtra["status"])
+        assertEquals("reason must match", "Qualified", enrolledExtra["reason"])
+        assertEquals("errorString must match", null, enrolledExtra["error_string"])
+        assertEquals("conflictSlug must match", null, enrolledExtra["conflict_slug"])
+    }
 }
 
 // Mocking utilities, from mozilla.components.support.test

components/nimbus/examples/experiment.rs

@@ -12,12 +12,21 @@ fn main() -> Result<()> {
     use clap::{App, Arg, SubCommand};
     use env_logger::Env;
     use nimbus::{
+        metrics::{EnrollmentStatusExtraDef, MetricsHandler},
         AppContext, AvailableRandomizationUnits, EnrollmentStatus, NimbusClient,
         NimbusTargetingHelper, RemoteSettingsConfig,
     };
     use std::collections::HashMap;
     use std::io::prelude::*;
 
+    pub struct NoopMetricsHandler;
+
+    impl MetricsHandler for NoopMetricsHandler {
+        fn record_enrollment_statuses(&self, _: Vec<EnrollmentStatusExtraDef>) {
+            // do nothing
+        }
+    }
+
     // We set the logging level to be `warn` here, meaning that only
     // logs of `warn` or higher will be actually be shown, any other
     // error will be omitted
@@ -208,6 +217,7 @@ fn main() -> Result<()> {
         db_path,
         Some(config),
         aru,
+        Box::new(NoopMetricsHandler),
     )?;
     log::info!("Nimbus ID is {}", nimbus_client.nimbus_id()?);
 

components/nimbus/ios/Nimbus/NimbusCreate.swift

@@ -3,6 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 import Foundation
+import Glean
 import UIKit
 
 private let logTag = "Nimbus.swift"
@@ -18,6 +19,22 @@ public let defaultErrorReporter: NimbusErrorReporter = { err in
     }
 }
 
+class GleanMetricsHandler: MetricsHandler {
+    func recordEnrollmentStatuses(enrollmentStatusExtras: [EnrollmentStatusExtraDef]) {
+        for extra in enrollmentStatusExtras {
+            GleanMetrics.NimbusEvents.enrollmentStatus
+                .record(GleanMetrics.NimbusEvents.EnrollmentStatusExtra(
+                    branch: extra.branch,
+                    conflictSlug: extra.conflictSlug,
+                    errorString: extra.errorString,
+                    reason: extra.reason,
+                    slug: extra.slug,
+                    status: extra.status
+                ))
+        }
+    }
+}
+
 public extension Nimbus {
     /// Create an instance of `Nimbus`.
     ///
@@ -63,7 +80,8 @@ public extension Nimbus {
                 userId: nil,
                 nimbusId: nil,
                 dummy: 0
-            )
+            ),
+            metricsHandler: GleanMetricsHandler()
         )
 
         return Nimbus(nimbusClient: nimbusClient, resourceBundles: resourceBundles, errorReporter: errorReporter)

components/nimbus/metrics.yaml

@@ -203,6 +203,39 @@ nimbus_events:
       - jhugman@mozilla.com
       - nimbus-team@mozilla.com
     expires: never
+  enrollment_status:
+    type: event
+    description: >
+      Recorded for each enrollment status each time the SDK completes application of pending experiments.
+    extra_keys:
+      slug:
+        type: string
+        description: The slug/unique identifier of the experiment
+      status:
+        type: string
+        description: The status of this enrollment
+      reason:
+        type: string
+        description: The reason the client is in the noted status
+      branch:
+        type: string
+        description: The branch slug/identifier that was randomly chosen (if the client is enrolled)
+      error_string:
+        type: string
+        description: If the enrollment resulted in an error, the associated error string
+      conflict_slug:
+        type: string
+        description: If the enrollment hit a feature conflict, the slug of the conflicting experiment/rollout
+    bugs:
+      - https://mozilla-hub.atlassian.net/browse/EXP-3827
+    data_reviews:
+      - ''
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - chumphreys@mozilla.com
+      - project-nimbus@mozilla.com
+    expires: never
 nimbus_health:
   cache_not_ready_for_feature:
     type: event

components/nimbus/src/enrollment.rs

@@ -12,6 +12,7 @@ use ::uuid::Uuid;
 use serde_derive::*;
 use std::{
     collections::{HashMap, HashSet},
+    fmt::{Display, Formatter, Result as FmtResult},
     time::{Duration, SystemTime, UNIX_EPOCH},
 };
 
@@ -28,6 +29,18 @@ pub enum EnrolledReason {
     OptIn,
 }
 
+impl Display for EnrolledReason {
+    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
+        Display::fmt(
+            match self {
+                EnrolledReason::Qualified => "Qualified",
+                EnrolledReason::OptIn => "OptIn",
+            },
+            f,
+        )
+    }
+}
+
 // These are types we use internally for managing non-enrollments.
 
 // ⚠️ Attention : Changes to this type should be accompanied by a new test  ⚠️
@@ -46,6 +59,21 @@ pub enum NotEnrolledReason {
     FeatureConflict,
 }
 
+impl Display for NotEnrolledReason {
+    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
+        Display::fmt(
+            match self {
+                NotEnrolledReason::OptOut => "OptOut",
+                NotEnrolledReason::NotSelected => "NotSelected",
+                NotEnrolledReason::NotTargeted => "NotTargeted",
+                NotEnrolledReason::EnrollmentsPaused => "EnrollmentsPaused",
+                NotEnrolledReason::FeatureConflict => "FeatureConflict",
+            },
+            f,
+        )
+    }
+}
+
 // These are types we use internally for managing disqualifications.
 
 // ⚠️ Attention : Changes to this type should be accompanied by a new test  ⚠️
@@ -62,6 +90,20 @@ pub enum DisqualifiedReason {
     NotSelected,
 }
 
+impl Display for DisqualifiedReason {
+    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
+        Display::fmt(
+            match self {
+                DisqualifiedReason::Error => "Error",
+                DisqualifiedReason::OptOut => "OptOut",
+                DisqualifiedReason::NotSelected => "NotSelected",
+                DisqualifiedReason::NotTargeted => "NotTargeted",
+            },
+            f,
+        )
+    }
+}
+
 // Every experiment has an ExperimentEnrollment, even when we aren't enrolled.
 
 // ⚠️ Attention : Changes to this type should be accompanied by a new test  ⚠️
@@ -439,7 +481,9 @@ impl ExperimentEnrollment {
                 },
                 EnrollmentChangeEventType::Disqualification,
             ),
-            EnrollmentStatus::NotEnrolled { .. } | EnrollmentStatus::Error { .. } => unreachable!(),
+            EnrollmentStatus::NotEnrolled { .. } | EnrollmentStatus::Error { .. } => {
+                unreachable!()
+            }
         }
     }
 
@@ -496,6 +540,19 @@ pub enum EnrollmentStatus {
     },
 }
 
+impl EnrollmentStatus {
+    pub fn name(&self) -> String {
+        match self {
+            EnrollmentStatus::Enrolled { .. } => "Enrolled",
+            EnrollmentStatus::NotEnrolled { .. } => "NotEnrolled",
+            EnrollmentStatus::Disqualified { .. } => "Disqualified",
+            EnrollmentStatus::WasEnrolled { .. } => "WasEnrolled",
+            EnrollmentStatus::Error { .. } => "Error",
+        }
+        .into()
+    }
+}
+
 impl EnrollmentStatus {
     // Note that for now, we only support a single feature_id per experiment,
     // so this code is expected to shift once we start supporting multiple.

components/nimbus/src/error.rs

@@ -66,6 +66,9 @@ pub enum NimbusError {
     #[cfg(not(feature = "stateful"))]
     #[error("Error in Cirrus: {0}")]
     CirrusError(#[from] CirrusClientError),
+    #[cfg(feature = "stateful")]
+    #[error("UniFFI callback error: {0}")]
+    UniFFICallbackError(#[from] uniffi::UnexpectedUniFFICallbackError),
 }
 
 #[cfg(feature = "stateful")]

components/nimbus/src/lib.rs

@@ -2,6 +2,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+extern crate core;
+
 mod defaults;
 mod enrollment;
 mod evaluator;
@@ -23,6 +25,8 @@ pub use targeting::NimbusTargetingHelper;
 
 cfg_if::cfg_if! {
     if #[cfg(feature = "stateful")] {
+        pub mod metrics;
+
         pub mod stateful;
 
         pub use stateful::nimbus_client::*;