Feature/#6557 Organic search telemetry

app/metrics.yaml

@@ -2030,6 +2030,20 @@ browser.search:
       notification_emails:
         - fenix-core@mozilla.com
       expires: "2020-09-01"
+    in_content:
+      type: labeled_counter
+      description: >
+        Records the type of interaction a user has on SERP pages.
+      send_in_pings:
+        - baseline
+        - metrics
+      bugs:
+        - https://github.com/mozilla-mobile/fenix/issues/6557
+      data_reviews:
+        - https://github.com/mozilla-mobile/fenix/pull/10167
+      notification_emails:
+        - fenix-core@mozilla.com
+      expires: "2020-09-01"
 
 addons:
   open_addons_in_settings:

app/src/main/assets/extensions/cookies/cookies.js

@@ -0,0 +1,47 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const COOKIES_CHECK_TIMEOUT_MS = 1000;
+
+function sendCookies(cookies) {
+    let message = {
+     'url': document.location.href,
+     'cookies': cookies
+    }
+    browser.runtime.sendNativeMessage("BrowserCookiesMessage", message);
+}
+
+function notify(message) {
+   sendCookies(message.cookies);
+}
+
+browser.runtime.onMessage.addListener(notify);
+
+const events = ["pageshow", "load", "unload"];
+var timeout;
+
+const eventLogger = event => {
+  switch (event.type) {
+    case "load":
+        timeout = setTimeout(() => {
+            browser.runtime.sendMessage({"checkCookies": true});
+        }, COOKIES_CHECK_TIMEOUT_MS);
+        break;
+    case "pageshow":
+        if (event.persisted) {
+          timeout = setTimeout(() => {
+              browser.runtime.sendMessage({"checkCookies": true});
+          }, COOKIES_CHECK_TIMEOUT_MS);
+        }
+        break;
+    case "unload":
+        clearTimeout(timeout);
+    default:
+        console.log('Event:', event.type);
+  }
+};
+
+events.forEach(eventName =>
+  window.addEventListener(eventName, eventLogger)
+);

app/src/main/assets/extensions/cookies/cookiesBackground.js

@@ -0,0 +1,28 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+browser.runtime.onMessage.addListener(notify);
+
+function sendMessageToTabs(tabs, cookies) {
+  for (let tab of tabs) {
+    browser.tabs.sendMessage(
+      tab.id,
+      {cookies: cookies}
+    );
+  }
+}
+
+function notify(message) {
+    if(message.checkCookies) {
+        browser.cookies.getAll({})
+            .then(cookies => {
+                browser.tabs.query({
+                    currentWindow: true,
+                    active: true
+                }).then(tabs => {
+                    sendMessageToTabs(tabs, cookies);
+                });
+        });
+    }
+}

app/src/main/assets/extensions/cookies/manifest.json

@@ -0,0 +1,32 @@
+{
+  "manifest_version": 2,
+  "name": "Mozilla Android Components - Cookies",
+  "version": "1.0",
+  "content_scripts": [
+    {
+      "matches": ["https://*/*"],
+      "include_globs": [
+        "https://www.google.*/search*",
+        "https://www.baidu.com/from=844b/s*",
+        "https://www.baidu.com/from=844b/baidu*",
+        "https://*search.yahoo.com/search*",
+        "https://www.bing.com/search*",
+        "https://duckduckgo.com/*"
+      ],
+      "js": ["cookies.js"],
+      "run_at": "document_end"
+    }
+  ],
+  "background": {
+    "scripts": ["cookiesBackground.js"]
+  },
+  "permissions": [
+    "geckoViewAddons",
+    "nativeMessaging",
+    "webNavigation",
+    "webRequest",
+    "webRequestBlocking",
+    "cookies",
+    "*://*/*"
+  ]
+}

app/src/main/java/org/mozilla/fenix/browser/UriOpenedObserver.kt

@@ -9,10 +9,10 @@ import androidx.fragment.app.FragmentActivity
 import androidx.lifecycle.LifecycleOwner
 import mozilla.components.browser.session.Session
 import mozilla.components.browser.session.SessionManager
-import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry
 import org.mozilla.fenix.components.metrics.MetricController
 import org.mozilla.fenix.ext.components
 import org.mozilla.fenix.ext.metrics
+import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry
 
 class UriOpenedObserver(
     private val owner: LifecycleOwner,
@@ -25,7 +25,7 @@ class UriOpenedObserver(
         activity,
         activity.components.core.sessionManager,
         activity.metrics,
-        activity.components.core.ads
+        activity.components.core.adsTelemetry
     )
 
     @VisibleForTesting

app/src/main/java/org/mozilla/fenix/components/Core.kt

@@ -42,10 +42,11 @@ import org.mozilla.fenix.AppRequestInterceptor
 import org.mozilla.fenix.Config
 import org.mozilla.fenix.HomeActivity
 import org.mozilla.fenix.R
-import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry
 import org.mozilla.fenix.ext.components
 import org.mozilla.fenix.ext.settings
 import org.mozilla.fenix.media.MediaService
+import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry
+import org.mozilla.fenix.search.telemetry.incontent.InContentTelemetry
 import org.mozilla.fenix.utils.Mockable
 import java.util.concurrent.TimeUnit
 
@@ -125,12 +126,14 @@ class Core(private val context: Context) {
      */
     val sessionManager by lazy {
         SessionManager(engine, store).also { sessionManager ->
-
             // Install the "icons" WebExtension to automatically load icons for every visited website.
             icons.install(engine, store)
 
             // Install the "ads" WebExtension to get the links in an partner page.
-            ads.install(engine, store)
+            adsTelemetry.install(engine, store)
+
+            // Install the "cookies" WebExtension and tracks user interaction with SERPs.
+            searchTelemetry.install(engine, store)
 
             // Show an ongoing notification when recording devices (camera, microphone) are used by web content
             RecordingDevicesNotificationFeature(context, sessionManager)
@@ -175,10 +178,14 @@ class Core(private val context: Context) {
         BrowserIcons(context, client)
     }
 
-    val ads by lazy {
+    val adsTelemetry by lazy {
         AdsTelemetry(context.components.analytics.metrics)
     }
 
+    val searchTelemetry by lazy {
+        InContentTelemetry(context.components.analytics.metrics)
+    }
+
     /**
      * Shortcut component for managing shortcuts on the device home screen.
      */

app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt

@@ -117,6 +117,11 @@ private val Event.wrapper: EventWrapper<*>?
                 BrowserSearch.adClicks[label].add(1)
             }
         )
+        is Event.SearchInContent -> EventWrapper<NoExtraKeys>(
+            {
+                BrowserSearch.inContent[label].add(1)
+            }
+        )
         is Event.SearchShortcutSelected -> EventWrapper(
             { SearchShortcuts.selected.record(it) },
             { SearchShortcuts.selectedKeys.valueOf(it) }

app/src/main/java/org/mozilla/fenix/components/metrics/Metrics.kt

@@ -375,6 +375,11 @@ sealed class Event {
             get() = providerName
     }
 
+    data class SearchInContent(val keyName: String) : Event() {
+        val label: String
+            get() = keyName
+    }
+
     class ContextMenuItemTapped private constructor(val item: String) : Event() {
         override val extras: Map<ContextMenu.itemTappedKeys, String>?
             get() = mapOf(ContextMenu.itemTappedKeys.named to item)

app/src/main/java/org/mozilla/fenix/search/telemetry/BaseSearchTelemetry.kt

@@ -0,0 +1,159 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.search.telemetry
+
+import androidx.annotation.VisibleForTesting
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.map
+import mozilla.components.browser.state.state.BrowserState
+import mozilla.components.browser.state.store.BrowserStore
+import mozilla.components.concept.engine.Engine
+import mozilla.components.concept.engine.EngineSession
+import mozilla.components.concept.engine.webextension.MessageHandler
+import mozilla.components.concept.engine.webextension.WebExtension
+import mozilla.components.lib.state.ext.flowScoped
+import mozilla.components.support.base.log.logger.Logger
+import mozilla.components.support.ktx.android.org.json.toList
+import mozilla.components.support.ktx.kotlinx.coroutines.flow.filterChanged
+import org.json.JSONObject
+
+abstract class BaseSearchTelemetry {
+
+    @VisibleForTesting
+    internal val providerList = listOf(
+        SearchProviderModel(
+            name = "google",
+            regexp = "^https:\\/\\/www\\.google\\.(?:.+)\\/search",
+            queryParam = "q",
+            codeParam = "client",
+            codePrefixes = listOf("firefox"),
+            followOnParams = listOf("oq", "ved", "ei"),
+            extraAdServersRegexps = listOf("^https?:\\/\\/www\\.google(?:adservices)?\\.com\\/(?:pagead\\/)?aclk")
+        ),
+        SearchProviderModel(
+            name = "duckduckgo",
+            regexp = "^https:\\/\\/duckduckgo\\.com\\/",
+            queryParam = "q",
+            codeParam = "t",
+            codePrefixes = listOf("f"),
+            extraAdServersRegexps = listOf(
+                "^https:\\/\\/duckduckgo.com\\/y\\.js",
+                "^https:\\/\\/www\\.amazon\\.(?:[a-z.]{2,24}).*(?:tag=duckduckgo-)"
+            )
+        ),
+        SearchProviderModel(
+            name = "yahoo",
+            regexp = "^https:\\/\\/(?:.*)search\\.yahoo\\.com\\/search",
+            queryParam = "p"
+        ),
+        SearchProviderModel(
+            name = "baidu",
+            regexp = "^https:\\/\\/www\\.baidu\\.com\\/from=844b\\/(?:s|baidu)",
+            queryParam = "wd",
+            codeParam = "tn",
+            codePrefixes = listOf("34046034_", "monline_"),
+            followOnParams = listOf("oq")
+        ),
+        SearchProviderModel(
+            name = "bing",
+            regexp = "^https:\\/\\/www\\.bing\\.com\\/search",
+            queryParam = "q",
+            codeParam = "pc",
+            codePrefixes = listOf("MOZ", "MZ"),
+            followOnCookies = listOf(
+                SearchProviderCookie(
+                    extraCodeParam = "form",
+                    extraCodePrefixes = listOf("QBRE"),
+                    host = "www.bing.com",
+                    name = "SRCHS",
+                    codeParam = "PC",
+                    codePrefixes = listOf("MOZ", "MZ")
+                )
+            ),
+            extraAdServersRegexps = listOf(
+                "^https:\\/\\/www\\.bing\\.com\\/acli?c?k",
+                "^https:\\/\\/www\\.bing\\.com\\/fd\\/ls\\/GLinkPingPost\\.aspx.*acli?c?k"
+            )
+        )
+    )
+
+    abstract fun install(engine: Engine, store: BrowserStore)
+
+    internal fun getProviderForUrl(url: String): SearchProviderModel? {
+        for (provider in providerList) {
+            if (Regex(provider.regexp).containsMatchIn(url)) {
+                return provider
+            }
+        }
+        return null
+    }
+
+    internal fun installWebExtension(
+        engine: Engine,
+        store: BrowserStore,
+        extensionInfo: ExtensionInfo
+    ) {
+        engine.installWebExtension(
+            id = extensionInfo.id,
+            url = extensionInfo.resourceUrl,
+            allowContentMessaging = true,
+            onSuccess = { extension ->
+                store.flowScoped { flow -> subscribeToUpdates(flow, extension, extensionInfo) }
+            },
+            onError = { _, throwable ->
+                Logger.error("Could not install ${extensionInfo.id} extension", throwable)
+            })
+    }
+
+    private suspend fun subscribeToUpdates(
+        flow: Flow<BrowserState>,
+        extension: WebExtension,
+        extensionInfo: ExtensionInfo
+    ) {
+        // Whenever we see a new EngineSession in the store then we register our content message
+        // handler if it has not been added yet.
+        flow.map { it.tabs }
+            .filterChanged { it.engineState.engineSession }
+            .collect { state ->
+                val engineSession = state.engineState.engineSession ?: return@collect
+
+                if (extension.hasContentMessageHandler(engineSession, extensionInfo.messageId)) {
+                    return@collect
+                }
+                extension.registerContentMessageHandler(
+                    engineSession,
+                    extensionInfo.messageId,
+                    SearchTelemetryMessageHandler()
+                )
+            }
+    }
+
+    protected fun <T> getMessageList(message: JSONObject, key: String): List<T> {
+        return message.getJSONArray(key).toList()
+    }
+
+    /**
+     * This method is used to process any valid json message coming from a web-extension
+     */
+    @VisibleForTesting
+    internal abstract fun processMessage(message: JSONObject)
+
+    @VisibleForTesting
+    internal inner class SearchTelemetryMessageHandler : MessageHandler {
+
+        override fun onMessage(message: Any, source: EngineSession?): Any? {
+            if (message is JSONObject) {
+                processMessage(message)
+            } else {
+                throw IllegalStateException("Received unexpected message: $message")
+            }
+
+            // Needs to return something that is not null and not Unit:
+            // https://github.com/mozilla-mobile/android-components/issues/2969
+            return ""
+        }
+    }
+}

app/src/main/java/org/mozilla/fenix/search/telemetry/ExtensionInfo.kt

@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.search.telemetry
+
+data class ExtensionInfo(
+    internal val id: String,
+    internal val resourceUrl: String,
+    internal val messageId: String
+)