For #18036 - Add TrackKeyInfo to Ad Click Metrics (#18159)

app/src/main/assets/extensions/ads/ads.js

@@ -14,28 +14,39 @@ function collectLinks(urls) {
     }
 }
 
-function sendLinks() {
+function sendLinks(cookies) {
     let urls = [];
     collectLinks(urls);
 
     let message = {
      'url': document.location.href,
-     'urls': urls
+     'urls': urls,
+     'cookies': cookies
     };
     browser.runtime.sendNativeMessage("MozacBrowserAds", message);
 }
 
+function notify(message) {
+   sendLinks(message.cookies);
+}
+
+browser.runtime.onMessage.addListener(notify);
+
 const events = ["pageshow", "load", "unload"];
 var timeout;
 
 const eventLogger = event => {
   switch (event.type) {
     case "load":
-        timeout = setTimeout(sendLinks, ADLINK_CHECK_TIMEOUT_MS);
+        timeout = setTimeout(() => {
+            browser.runtime.sendMessage({ "checkCookies": true });
+        }, ADLINK_CHECK_TIMEOUT_MS)
         break;
     case "pageshow":
         if (event.persisted) {
-          timeout = setTimeout(sendLinks, ADLINK_CHECK_TIMEOUT_MS);
+          timeout = setTimeout(() => {
+              browser.runtime.sendMessage({ "checkCookies": true });
+          }, ADLINK_CHECK_TIMEOUT_MS)
         }
         break;
     case "unload":

app/src/main/assets/extensions/ads/adsBackground.js

@@ -0,0 +1,28 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+browser.runtime.onMessage.addListener(notify);
+
+function sendMessageToTabs(tabs, cookies) {
+  for (let tab of tabs) {
+    browser.tabs.sendMessage(
+      tab.id,
+      { cookies }
+    );
+  }
+}
+
+function notify(message) {
+    if (message.checkCookies) {
+        browser.cookies.getAll({})
+            .then(cookies => {
+                browser.tabs.query({
+                    currentWindow: true,
+                    active: true
+                }).then(tabs => {
+                    sendMessageToTabs(tabs, cookies);
+                });
+            });
+    }
+}

app/src/main/assets/extensions/ads/manifest.template.json

@@ -19,9 +19,20 @@
       "run_at": "document_end"
     }
   ],
+  "background": {
+    "scripts": ["adsBackground.js"]
+  },
   "permissions": [
     "geckoViewAddons",
     "nativeMessaging",
-    "nativeMessagingFromContent"
+    "nativeMessagingFromContent",
+    "geckoViewAddons",
+    "nativeMessaging",
+    "nativeMessagingFromContent",
+    "webNavigation",
+    "webRequest",
+    "webRequestBlocking",
+    "cookies",
+    "*://*/*"
   ]
 }

app/src/main/java/org/mozilla/fenix/components/metrics/Event.kt

@@ -515,9 +515,9 @@ sealed class Event {
             get() = providerName
     }
 
-    data class SearchAdClicked(val providerName: String) : Event() {
+    data class SearchAdClicked(val keyName: String) : Event() {
         val label: String
-            get() = providerName
+            get() = keyName
     }
 
     data class SearchInContent(val keyName: String) : Event() {

app/src/main/java/org/mozilla/fenix/search/telemetry/SearchProviderModel.kt

@@ -39,7 +39,7 @@ data class SearchProviderModel(
      * Checks if any of the given URLs represent an ad from the search engine.
      * Used to check if a clicked link was for an ad.
      */
-    fun containsAds(urlList: List<String>) = urlList.any { url -> isAd(url) }
+    fun containsAdLinks(urlList: List<String>) = urlList.any { url -> isAd(url) }
 
     private fun isAd(url: String) =
         extraAdServersRegexps.any { adsRegex -> adsRegex.containsMatchIn(url) }

app/src/main/java/org/mozilla/fenix/search/telemetry/TrackKeyInfo.kt

@@ -0,0 +1,36 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.search.telemetry
+
+import java.util.Locale
+
+/**
+ * A data class that tracks key information about a Search Engine Result Page (SERP).
+ *
+ * @property provider The name of the search provider.
+ * @property type The search access point type (SAP). This is either "organic", "sap" or
+ * "sap-follow-on".
+ * @property code The search URL's `code` query parameter.
+ * @property channel The search URL's `channel` query parameter.
+ */
+internal data class TrackKeyInfo(
+    var provider: String,
+    var type: String,
+    var code: String?,
+    var channel: String? = null
+) {
+    /**
+     * Returns the track key information into the following string format:
+     * `<provider>.in-content.[sap|sap-follow-on|organic].[code|none](.[channel])?`.
+     */
+    fun createTrackKey(): String {
+        return "${provider.toLowerCase(Locale.ROOT)}.in-content" +
+                ".${type.toLowerCase(Locale.ROOT)}" +
+                ".${code?.toLowerCase(Locale.ROOT) ?: "none"}" +
+                if (!channel?.toLowerCase(Locale.ROOT).isNullOrBlank())
+                    ".${channel?.toLowerCase(Locale.ROOT)}"
+                else ""
+    }
+}

app/src/main/java/org/mozilla/fenix/search/telemetry/Utils.kt

@@ -0,0 +1,96 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.search.telemetry
+
+import android.net.Uri
+import org.json.JSONObject
+
+private const val SEARCH_TYPE_SAP_FOLLOW_ON = "sap-follow-on"
+private const val SEARCH_TYPE_SAP = "sap"
+private const val SEARCH_TYPE_ORGANIC = "organic"
+private const val CHANNEL_KEY = "channel"
+
+internal fun getTrackKey(
+    provider: SearchProviderModel,
+    uri: Uri,
+    cookies: List<JSONObject>
+): String {
+    val paramSet = uri.queryParameterNames
+    var code: String? = null
+
+    if (provider.codeParam.isNotEmpty()) {
+        code = uri.getQueryParameter(provider.codeParam)
+
+        // Try cookies first because Bing has followOnCookies and valid code, but no
+        // followOnParams => would tracks organic instead of sap-follow-on
+        if (provider.followOnCookies.isNotEmpty()) {
+            // Checks if engine contains a valid follow-on cookie, otherwise return default
+            getTrackKeyFromCookies(provider, uri, cookies)?.let {
+                return it.createTrackKey()
+            }
+        }
+
+        // For Bing if it didn't have a valid cookie and for all the other search engines
+        if (hasValidCode(code, provider)) {
+            val channel = uri.getQueryParameter(CHANNEL_KEY)
+            val type = getSapType(provider.followOnParams, paramSet)
+            return TrackKeyInfo(provider.name, type, code, channel).createTrackKey()
+        }
+    }
+
+    // Default to organic search type if no code parameter was found.
+    return TrackKeyInfo(provider.name, SEARCH_TYPE_ORGANIC, code).createTrackKey()
+}
+
+private fun getTrackKeyFromCookies(
+    provider: SearchProviderModel,
+    uri: Uri,
+    cookies: List<JSONObject>
+): TrackKeyInfo? {
+    // Especially Bing requires lots of extra work related to cookies.
+    for (followOnCookie in provider.followOnCookies) {
+        val eCode = uri.getQueryParameter(followOnCookie.extraCodeParam)
+        if (eCode == null || !followOnCookie.extraCodePrefixes.any { prefix ->
+                eCode.startsWith(prefix)
+            }) {
+            continue
+        }
+
+        // If this cookie is present, it's probably an SAP follow-on.
+        // This might be an organic follow-on in the same session, but there
+        // is no way to tell the difference.
+        for (cookie in cookies) {
+            if (cookie.getString("name") != followOnCookie.name) {
+                continue
+            }
+            val valueList = cookie.getString("value")
+                .split("=")
+                .map { item -> item.trim() }
+
+            if (valueList.size == 2 && valueList[0] == followOnCookie.codeParam &&
+                followOnCookie.codePrefixes.any { prefix ->
+                    valueList[1].startsWith(
+                        prefix
+                    )
+                }
+            ) {
+                return TrackKeyInfo(provider.name, SEARCH_TYPE_SAP_FOLLOW_ON, valueList[1])
+            }
+        }
+    }
+
+    return null
+}
+
+private fun getSapType(followOnParams: List<String>, paramSet: Set<String>): String {
+    return if (followOnParams.any { param -> paramSet.contains(param) }) {
+        SEARCH_TYPE_SAP_FOLLOW_ON
+    } else {
+        SEARCH_TYPE_SAP
+    }
+}
+
+private fun hasValidCode(code: String?, provider: SearchProviderModel): Boolean =
+    code != null && provider.codePrefixes.any { prefix -> code.startsWith(prefix) }

app/src/main/java/org/mozilla/fenix/search/telemetry/ads/AdsTelemetry.kt

@@ -5,16 +5,22 @@
 package org.mozilla.fenix.search.telemetry.ads
 
 import androidx.annotation.VisibleForTesting
+import androidx.core.net.toUri
 import mozilla.components.browser.state.store.BrowserStore
 import mozilla.components.concept.engine.Engine
 import org.json.JSONObject
 import org.mozilla.fenix.components.metrics.Event
 import org.mozilla.fenix.components.metrics.MetricController
 import org.mozilla.fenix.search.telemetry.BaseSearchTelemetry
 import org.mozilla.fenix.search.telemetry.ExtensionInfo
+import org.mozilla.fenix.search.telemetry.getTrackKey
 
 class AdsTelemetry(private val metrics: MetricController) : BaseSearchTelemetry() {
 
+    // Cache the cookies provided by the ADS_EXTENSION_ID extension to be used when tracking
+    // the Ads clicked telemetry.
+    var cachedCookies = listOf<JSONObject>()
+
     override fun install(
         engine: Engine,
         store: BrowserStore
@@ -27,38 +33,64 @@ class AdsTelemetry(private val metrics: MetricController) : BaseSearchTelemetry(
         installWebExtension(engine, store, info)
     }
 
-    fun trackAdClickedMetric(sessionUrl: String?, urlPath: List<String>) {
-        if (sessionUrl == null) {
-            return
-        }
-        val provider = getProviderForUrl(sessionUrl)
-        provider?.let {
-            if (it.containsAds(urlPath)) {
-                metrics.track(Event.SearchAdClicked(it.name))
-            }
-        }
-    }
-
     override fun processMessage(message: JSONObject) {
+        // Cache the cookies list when the extension sends a message.
+        cachedCookies = getMessageList(
+            message,
+            ADS_MESSAGE_COOKIES_KEY
+        )
+
         val urls = getMessageList<String>(message, ADS_MESSAGE_DOCUMENT_URLS_KEY)
         val provider = getProviderForUrl(message.getString(ADS_MESSAGE_SESSION_URL_KEY))
+
         provider?.let {
-            if (it.containsAds(urls)) {
+            if (it.containsAdLinks(urls)) {
                 metrics.track(Event.SearchWithAds(it.name))
             }
         }
     }
 
+    /**
+     * If a search ad is clicked, record the search ad that was clicked. This method is called
+     * when the browser is navigating to a new URL, which may be a search ad.
+     *
+     * @param url The URL of the page before the search ad was clicked. This is used to determine
+     * the originating search provider.
+     * @param urlPath A list of the URLs and load requests collected in between location changes.
+     * Clicking on a search ad generates a list of redirects from the originating search provider
+     * to the ad source. This is used to determine if there was an ad click.
+     */
+    fun trackAdClickedMetric(url: String?, urlPath: List<String>) {
+        val uri = url?.toUri() ?: return
+        val provider = getProviderForUrl(url) ?: return
+        val paramSet = uri.queryParameterNames
+
+        if (!paramSet.contains(provider.queryParam) || !provider.containsAdLinks(urlPath)) {
+            // Do nothing if the URL does not have the search provider's query parameter or
+            // there were no ad clicks.
+            return
+        }
+
+        metrics.track(Event.SearchAdClicked(getTrackKey(provider, uri, cachedCookies)))
+    }
+
     companion object {
         @VisibleForTesting
         internal const val ADS_EXTENSION_ID = "ads@mozac.org"
+
         @VisibleForTesting
         internal const val ADS_EXTENSION_RESOURCE_URL = "resource://android/assets/extensions/ads/"
+
         @VisibleForTesting
         internal const val ADS_MESSAGE_SESSION_URL_KEY = "url"
+
         @VisibleForTesting
         internal const val ADS_MESSAGE_DOCUMENT_URLS_KEY = "urls"
+
         @VisibleForTesting
         internal const val ADS_MESSAGE_ID = "MozacBrowserAds"
+
+        @VisibleForTesting
+        internal const val ADS_MESSAGE_COOKIES_KEY = "cookies"
     }
 }